From f2efed9608cfa192fde5e0a3fc455792fb6911c2 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Mon, 8 Jun 2015 20:00:45 +0800 Subject: [PATCH 001/487] add manyuser branch support udp over tcp support chacha20 & salsa20 (base on libsodium) --- Config.py | 12 ++ __init__.py | 1 + asyncmgr.py | 100 +++++++++ config.json | 12 ++ db_transfer.py | 148 +++++++++++++ server.py | 22 ++ server_pool.py | 211 +++++++++++++++++++ setup.py | 39 ---- shadowsocks.sql | 24 +++ shadowsocks/common.py | 4 +- shadowsocks/crypto/ctypes_libsodium.py | 135 ++++++++++++ shadowsocks/crypto/ctypes_openssl.py | 188 +++++++++++++++++ shadowsocks/crypto/m2.py | 117 +++++++++++ shadowsocks/encrypt.py | 2 + shadowsocks/tcprelay.py | 88 ++++++-- shadowsocks/udprelay.py | 1 + shadowsocks/utils.py | 278 +++++++++++++++++++++++++ switchrule.py | 2 + 18 files changed, 1327 insertions(+), 57 deletions(-) create mode 100644 Config.py create mode 100644 __init__.py create mode 100644 asyncmgr.py create mode 100644 config.json create mode 100644 db_transfer.py create mode 100644 server.py create mode 100644 server_pool.py delete mode 100644 setup.py create mode 100644 shadowsocks.sql create mode 100644 shadowsocks/crypto/ctypes_libsodium.py create mode 100644 shadowsocks/crypto/ctypes_openssl.py create mode 100644 shadowsocks/crypto/m2.py create mode 100644 shadowsocks/utils.py create mode 100644 switchrule.py diff --git a/Config.py b/Config.py new file mode 100644 index 00000000..7a444e6f --- /dev/null +++ b/Config.py @@ -0,0 +1,12 @@ +#Config +MYSQL_HOST = 'mdss.mengsky.net' +MYSQL_PORT = 3306 +MYSQL_USER = 'ss' +MYSQL_PASS = 'ss' +MYSQL_DB = 'shadowsocks' + +MANAGE_PASS = 'ss233333333' +#if you want manage in other server you should set this value to global ip +MANAGE_BIND_IP = '127.0.0.1' +#make sure this port is idle +MANAGE_PORT = 23333 diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..013e4b7e --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/python diff --git a/asyncmgr.py b/asyncmgr.py new file mode 100644 index 00000000..8e24b421 --- /dev/null +++ b/asyncmgr.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import time +import os +import socket +import struct +import re +import logging +from shadowsocks import common +from shadowsocks import lru_cache +from shadowsocks import eventloop +import server_pool +import Config + +class ServerMgr(object): + + def __init__(self): + self._loop = None + self._request_id = 1 + self._hosts = {} + self._hostname_status = {} + self._hostname_to_cb = {} + self._cb_to_hostname = {} + self._last_time = time.time() + self._sock = None + self._servers = None + + def add_to_loop(self, loop): + if self._loop: + raise Exception('already add to loop') + self._loop = loop + # TODO when dns server is IPv6 + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + self._sock.bind((Config.MANAGE_BIND_IP, Config.MANAGE_PORT)) + self._sock.setblocking(False) + loop.add(self._sock, eventloop.POLL_IN) + loop.add_handler(self.handle_events) + + def _handle_data(self, sock): + data, addr = sock.recvfrom(128) + #manage pwd:port:passwd:action + args = data.split(':') + if len(args) < 4: + return + if args[0] == Config.MANAGE_PASS: + if args[3] == '0': + server_pool.ServerPool.get_instance().cb_del_server(args[1]) + elif args[3] == '1': + server_pool.ServerPool.get_instance().new_server(args[1], args[2]) + + def handle_events(self, events): + for sock, fd, event in events: + if sock != self._sock: + continue + if event & eventloop.POLL_ERR: + logging.error('mgr socket err') + self._loop.remove(self._sock) + self._sock.close() + # TODO when dns server is IPv6 + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + self._sock.setblocking(False) + self._loop.add(self._sock, eventloop.POLL_IN) + else: + self._handle_data(sock) + break + + def close(self): + if self._sock: + self._sock.close() + self._sock = None + + +def test(): + pass + +if __name__ == '__main__': + test() diff --git a/config.json b/config.json new file mode 100644 index 00000000..57e9cc18 --- /dev/null +++ b/config.json @@ -0,0 +1,12 @@ +{ + "server":"0.0.0.0", + "server_ipv6": "::", + "server_port":8388, + "local_address": "127.0.0.1", + "local_port":1080, + "password":"m", + "timeout":300, + "method":"aes-256-cfb", + "fast_open": false, + "workers": 1 +} diff --git a/db_transfer.py b/db_transfer.py new file mode 100644 index 00000000..8126d0a3 --- /dev/null +++ b/db_transfer.py @@ -0,0 +1,148 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +import logging +import cymysql +import time +import sys +from server_pool import ServerPool +import Config + +class DbTransfer(object): + + instance = None + + def __init__(self): + self.last_get_transfer = {} + + @staticmethod + def get_instance(): + if DbTransfer.instance is None: + DbTransfer.instance = DbTransfer() + return DbTransfer.instance + + def push_db_all_user(self): + #更新用户流量到数据库 + last_transfer = self.last_get_transfer + curr_transfer = ServerPool.get_instance().get_servers_transfer() + #上次和本次的增量 + dt_transfer = {} + for id in curr_transfer.keys(): + if id in last_transfer: + if last_transfer[id][0] == curr_transfer[id][0] and last_transfer[id][1] == curr_transfer[id][1]: + continue + elif curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + continue + elif last_transfer[id][0] <= curr_transfer[id][0] and \ + last_transfer[id][1] <= curr_transfer[id][1]: + dt_transfer[id] = [curr_transfer[id][0] - last_transfer[id][0], + curr_transfer[id][1] - last_transfer[id][1]] + else: + dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + else: + if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + continue + dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + + self.last_get_transfer = curr_transfer + query_head = 'UPDATE user' + query_sub_when = '' + query_sub_when2 = '' + query_sub_in = None + last_time = time.time() + for id in dt_transfer.keys(): + query_sub_when += ' WHEN %s THEN u+%s' % (id, dt_transfer[id][0]) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, dt_transfer[id][1]) + if query_sub_in is not None: + query_sub_in += ',%s' % id + else: + query_sub_in = '%s' % id + if query_sub_when == '': + return + query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ + ' END, d = CASE port' + query_sub_when2 + \ + ' END, t = ' + str(int(last_time)) + \ + ' WHERE port IN (%s)' % query_sub_in + #print query_sql + conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, + passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') + cur = conn.cursor() + cur.execute(query_sql) + cur.close() + conn.commit() + conn.close() + + @staticmethod + def pull_db_all_user(): + #数据库所有用户信息 + conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, + passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') + cur = conn.cursor() + cur.execute("SELECT port, u, d, transfer_enable, passwd, switch, enable, plan FROM user") + rows = [] + for r in cur.fetchall(): + rows.append(list(r)) + cur.close() + conn.close() + return rows + + @staticmethod + def del_server_out_of_bound_safe(last_rows, rows): + #停止超流量的服务 + #启动没超流量的服务 + #需要动态载入switchrule，以便实时修改规则 + cur_servers = {} + for row in rows: + try: + import switchrule + allow = switchrule.isTurnOn(row[7], row[5]) and row[6] == 1 and row[1] + row[2] < row[3] + except Exception, e: + allow = False + + cur_servers[row[0]] = row[4] + + if ServerPool.get_instance().server_is_run(row[0]) > 0: + if not allow: + logging.info('db stop server at port [%s]' % (row[0])) + ServerPool.get_instance().del_server(row[0]) + elif (row[0] in ServerPool.get_instance().tcp_servers_pool and ServerPool.get_instance().tcp_servers_pool[row[0]]._config['password'] != row[4]) \ + or (row[0] in ServerPool.get_instance().tcp_ipv6_servers_pool and ServerPool.get_instance().tcp_ipv6_servers_pool[row[0]]._config['password'] != row[4]): + #password changed + logging.info('db stop server at port [%s] reason: password changed' % (row[0])) + ServerPool.get_instance().del_server(row[0]) + elif ServerPool.get_instance().server_run_status(row[0]) is False: + if allow: + logging.info('db start server at port [%s] pass [%s]' % (row[0], row[4])) + ServerPool.get_instance().new_server(row[0], row[4]) + + for row in last_rows: + if row[0] in cur_servers: + if row[4] == cur_servers[row[0]]: + pass + else: + logging.info('db stop server at port [%s] reason: port not exist' % (row[0])) + ServerPool.get_instance().del_server(row[0]) + + @staticmethod + def thread_db(): + import socket + import time + timeout = 60 + socket.setdefaulttimeout(timeout) + last_rows = [] + while True: + #logging.warn('db loop') + + try: + DbTransfer.get_instance().push_db_all_user() + rows = DbTransfer.get_instance().pull_db_all_user() + DbTransfer.del_server_out_of_bound_safe(last_rows, rows) + last_rows = rows + except Exception as e: + logging.warn('db thread except:%s' % e) + finally: + time.sleep(15) + + +#SQLData.pull_db_all_user() +#print DbTransfer.get_instance().test() diff --git a/server.py b/server.py new file mode 100644 index 00000000..ad1e5583 --- /dev/null +++ b/server.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import time +import sys +import thread +import os +os.chdir(os.path.split(os.path.realpath(__file__))[0]) + +import server_pool +import db_transfer + +#def test(): +# thread.start_new_thread(DbTransfer.thread_db, ()) +# Api.web_server() + +if __name__ == '__main__': + #server_pool.ServerPool.get_instance() + #server_pool.ServerPool.get_instance().new_server(2333, '2333') + thread.start_new_thread(db_transfer.DbTransfer.thread_db, ()) + while True: + time.sleep(99999) diff --git a/server_pool.py b/server_pool.py new file mode 100644 index 00000000..c1232715 --- /dev/null +++ b/server_pool.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import logging +import time +from shadowsocks import utils +from shadowsocks import eventloop +from shadowsocks import tcprelay +from shadowsocks import udprelay +from shadowsocks import asyncdns +import thread +import threading +import sys +import asyncmgr +import Config +from socket import * + +class ServerPool(object): + + instance = None + + def __init__(self): + utils.check_python() + self.config = utils.get_config(False) + utils.print_shadowsocks() + self.dns_resolver = asyncdns.DNSResolver() + self.mgr = asyncmgr.ServerMgr() + self.udp_on = True ### UDP switch ===================================== + + self.tcp_servers_pool = {} + self.tcp_ipv6_servers_pool = {} + self.udp_servers_pool = {} + self.udp_ipv6_servers_pool = {} + + self.loop = eventloop.EventLoop() + thread.start_new_thread(ServerPool._loop, (self.loop, self.dns_resolver, self.mgr)) + + @staticmethod + def get_instance(): + if ServerPool.instance is None: + ServerPool.instance = ServerPool() + return ServerPool.instance + + @staticmethod + def _loop(loop, dns_resolver, mgr): + try: + mgr.add_to_loop(loop) + dns_resolver.add_to_loop(loop) + loop.run() + except (KeyboardInterrupt, IOError, OSError) as e: + logging.error(e) + import traceback + traceback.print_exc() + os.exit(0) + + def server_is_run(self, port): + port = int(port) + ret = 0 + if port in self.tcp_servers_pool: + ret = 1 + if port in self.tcp_ipv6_servers_pool: + ret |= 2 + return ret + + def server_run_status(self, port): + if 'server' in self.config: + if port not in self.tcp_servers_pool: + return False + if 'server_ipv6' in self.config: + if port not in self.tcp_ipv6_servers_pool: + return False + return True + + def new_server(self, port, password): + ret = True + port = int(port) + + if 'server_ipv6' in self.config: + if port in self.tcp_ipv6_servers_pool: + logging.info("server already at %s:%d" % (self.config['server_ipv6'], port)) + return 'this port server is already running' + else: + a_config = self.config.copy() + a_config['server'] = a_config['server_ipv6'] + a_config['server_port'] = port + a_config['password'] = password + try: + logging.info("starting server at %s:%d" % (a_config['server'], port)) + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) + tcp_server.add_to_loop(self.loop) + self.tcp_ipv6_servers_pool.update({port: tcp_server}) + if self.udp_on: + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) + udp_server.add_to_loop(self.loop) + self.udp_ipv6_servers_pool.update({port: udp_server}) + except Exception, e: + logging.warn("IPV6 exception") + logging.warn(e) + + if 'server' in self.config: + if port in self.tcp_servers_pool: + logging.info("server already at %s:%d" % (self.config['server'], port)) + return 'this port server is already running' + else: + a_config = self.config.copy() + a_config['server_port'] = port + a_config['password'] = password + try: + logging.info("starting server at %s:%d" % (a_config['server'], port)) + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) + tcp_server.add_to_loop(self.loop) + self.tcp_servers_pool.update({port: tcp_server}) + if self.udp_on: + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) + udp_server.add_to_loop(self.loop) + self.udp_servers_pool.update({port: udp_server}) + except Exception, e: + logging.warn("IPV4 exception") + logging.warn(e) + + return True + + def del_server(self, port): + port = int(port) + logging.info("del server at %d" % port) + try: + udpsock = socket(AF_INET, SOCK_DGRAM) + udpsock.sendto('%s:%s:0:0' % (Config.MANAGE_PASS, port), (Config.MANAGE_BIND_IP, Config.MANAGE_PORT)) + udpsock.close() + except Exception, e: + logging.warn(e) + return True + + def cb_del_server(self, port): + port = int(port) + + if port not in self.tcp_servers_pool: + logging.info("stopped server at %s:%d already stop" % (self.config['server'], port)) + else: + logging.info("stopped server at %s:%d" % (self.config['server'], port)) + try: + self.tcp_servers_pool[port].destroy() + del self.tcp_servers_pool[port] + except Exception, e: + logging.warn(e) + if self.udp_on: + try: + self.udp_servers_pool[port].destroy() + del self.udp_servers_pool[port] + except Exception, e: + logging.warn(e) + + if 'server_ipv6' in self.config: + if port not in self.tcp_ipv6_servers_pool: + logging.info("stopped server at %s:%d already stop" % (self.config['server_ipv6'], port)) + else: + logging.info("stopped server at %s:%d" % (self.config['server_ipv6'], port)) + try: + self.tcp_ipv6_servers_pool[port].destroy() + del self.tcp_ipv6_servers_pool[port] + except Exception, e: + logging.warn(e) + if self.udp_on: + try: + self.udp_ipv6_servers_pool[port].destroy() + del self.udp_ipv6_servers_pool[port] + except Exception, e: + logging.warn(e) + + return True + + def get_server_transfer(self, port): + port = int(port) + ret = [0, 0] + if port in self.tcp_servers_pool: + ret[0] = self.tcp_servers_pool[port].server_transfer_ul + ret[1] = self.tcp_servers_pool[port].server_transfer_dl + if port in self.tcp_ipv6_servers_pool: + ret[0] += self.tcp_ipv6_servers_pool[port].server_transfer_ul + ret[1] += self.tcp_ipv6_servers_pool[port].server_transfer_dl + return ret + + def get_servers_transfer(self): + servers = self.tcp_servers_pool.copy() + servers.update(self.tcp_ipv6_servers_pool) + ret = {} + for port in servers.keys(): + ret[port] = self.get_server_transfer(port) + return ret + diff --git a/setup.py b/setup.py deleted file mode 100644 index 07ea2db0..00000000 --- a/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -import codecs -from setuptools import setup - - -with codecs.open('README.rst', encoding='utf-8') as f: - long_description = f.read() - -setup( - name="shadowsocks", - version="2.6.11", - license='http://www.apache.org/licenses/LICENSE-2.0', - description="A fast tunnel proxy that help you get through firewalls", - author='clowwindy', - author_email='clowwindy42@gmail.com', - url='https://github.com/shadowsocks/shadowsocks', - packages=['shadowsocks', 'shadowsocks.crypto'], - package_data={ - 'shadowsocks': ['README.rst', 'LICENSE'] - }, - install_requires=[], - entry_points=""" - [console_scripts] - sslocal = shadowsocks.local:main - ssserver = shadowsocks.server:main - """, - classifiers=[ - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Internet :: Proxy Servers', - ], - long_description=long_description, -) diff --git a/shadowsocks.sql b/shadowsocks.sql new file mode 100644 index 00000000..8d84992d --- /dev/null +++ b/shadowsocks.sql @@ -0,0 +1,24 @@ +SET FOREIGN_KEY_CHECKS=0; + +CREATE TABLE `user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `email` varchar(32) NOT NULL, + `pass` varchar(16) NOT NULL, + `passwd` varchar(16) NOT NULL, + `t` int(11) NOT NULL DEFAULT '0', + `u` bigint(20) NOT NULL, + `d` bigint(20) NOT NULL, + `transfer_enable` bigint(20) NOT NULL, + `port` int(11) NOT NULL, + `switch` tinyint(4) NOT NULL DEFAULT '1', + `enable` tinyint(4) NOT NULL DEFAULT '1', + `type` tinyint(4) NOT NULL DEFAULT '1', + `last_get_gift_time` int(11) NOT NULL DEFAULT '0', + `last_rest_pass_time` int(11) NOT NULL DEFAULT '0', + PRIMARY KEY (`id`,`port`) +) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Records of user +-- ---------------------------- +INSERT INTO `user` VALUES ('7', 'test@test.com', '123456', '0000000', '1410609560', '0', '0', '9320666234', '50000', '1', '1', '7', '0', '0'); \ No newline at end of file diff --git a/shadowsocks/common.py b/shadowsocks/common.py index fc03d556..d740160b 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -144,6 +144,8 @@ def parse_header(data): dest_addr = None dest_port = None header_length = 0 + connecttype = (addrtype & 8) and 1 or 0 + addrtype &= ~8 if addrtype == ADDRTYPE_IPV4: if len(data) >= 7: dest_addr = socket.inet_ntoa(data[1:5]) @@ -175,7 +177,7 @@ def parse_header(data): 'encryption method' % addrtype) if dest_addr is None: return None - return addrtype, to_bytes(dest_addr), dest_port, header_length + return connecttype, to_bytes(dest_addr), dest_port, header_length class IPNetwork(object): diff --git a/shadowsocks/crypto/ctypes_libsodium.py b/shadowsocks/crypto/ctypes_libsodium.py new file mode 100644 index 00000000..efecfd41 --- /dev/null +++ b/shadowsocks/crypto/ctypes_libsodium.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import logging +from ctypes import CDLL, c_char_p, c_int, c_ulonglong, byref, \ + create_string_buffer, c_void_p + +__all__ = ['ciphers'] + +libsodium = None +loaded = False + +buf_size = 2048 + +# for salsa20 and chacha20 +BLOCK_SIZE = 64 + + +def load_libsodium(): + global loaded, libsodium, buf + + from ctypes.util import find_library + for p in ('sodium',): + libsodium_path = find_library(p) + if libsodium_path: + break + else: + raise Exception('libsodium not found') + logging.info('loading libsodium from %s', libsodium_path) + libsodium = CDLL(libsodium_path) + libsodium.sodium_init.restype = c_int + libsodium.crypto_stream_salsa20_xor_ic.restype = c_int + libsodium.crypto_stream_salsa20_xor_ic.argtypes = (c_void_p, c_char_p, + c_ulonglong, + c_char_p, c_ulonglong, + c_char_p) + libsodium.crypto_stream_chacha20_xor_ic.restype = c_int + libsodium.crypto_stream_chacha20_xor_ic.argtypes = (c_void_p, c_char_p, + c_ulonglong, + c_char_p, c_ulonglong, + c_char_p) + + libsodium.sodium_init() + + buf = create_string_buffer(buf_size) + loaded = True + + +class Salsa20Crypto(object): + def __init__(self, cipher_name, key, iv, op): + if not loaded: + load_libsodium() + self.key = key + self.iv = iv + self.key_ptr = c_char_p(key) + self.iv_ptr = c_char_p(iv) + if cipher_name == b'salsa20': + self.cipher = libsodium.crypto_stream_salsa20_xor_ic + elif cipher_name == b'chacha20': + self.cipher = libsodium.crypto_stream_chacha20_xor_ic + else: + raise Exception('Unknown cipher') + # byte counter, not block counter + self.counter = 0 + + def update(self, data): + global buf_size, buf + l = len(data) + + # we can only prepend some padding to make the encryption align to + # blocks + padding = self.counter % BLOCK_SIZE + if buf_size < padding + l: + buf_size = (padding + l) * 2 + buf = create_string_buffer(buf_size) + + if padding: + data = (b'\0' * padding) + data + self.cipher(byref(buf), c_char_p(data), padding + l, + self.iv_ptr, int(self.counter / BLOCK_SIZE), self.key_ptr) + self.counter += l + # buf is copied to a str object when we access buf.raw + # strip off the padding + return buf.raw[padding:padding + l] + + +ciphers = { + b'salsa20': (32, 8, Salsa20Crypto), + b'chacha20': (32, 8, Salsa20Crypto), +} + + +def test_salsa20(): + from shadowsocks.crypto import util + + cipher = Salsa20Crypto(b'salsa20', b'k' * 32, b'i' * 16, 1) + decipher = Salsa20Crypto(b'salsa20', b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + + +def test_chacha20(): + from shadowsocks.crypto import util + + cipher = Salsa20Crypto(b'chacha20', b'k' * 32, b'i' * 16, 1) + decipher = Salsa20Crypto(b'chacha20', b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + + +if __name__ == '__main__': + test_chacha20() + test_salsa20() diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py new file mode 100644 index 00000000..0ef8ce0f --- /dev/null +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import logging +from ctypes import CDLL, c_char_p, c_int, c_long, byref,\ + create_string_buffer, c_void_p + +__all__ = ['ciphers'] + +libcrypto = None +loaded = False + +buf_size = 2048 + + +def load_openssl(): + global loaded, libcrypto, buf + + from ctypes.util import find_library + for p in ('crypto', 'eay32', 'libeay32'): + libcrypto_path = find_library(p) + if libcrypto_path: + break + else: + raise Exception('libcrypto(OpenSSL) not found') + logging.info('loading libcrypto from %s', libcrypto_path) + libcrypto = CDLL(libcrypto_path) + libcrypto.EVP_get_cipherbyname.restype = c_void_p + libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p + + libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p, + c_char_p, c_char_p, c_int) + + libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p, + c_char_p, c_int) + + libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) + libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,) + if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'): + libcrypto.OpenSSL_add_all_ciphers() + + buf = create_string_buffer(buf_size) + loaded = True + + +def load_cipher(cipher_name): + func_name = b'EVP_' + cipher_name.replace(b'-', b'_') + if bytes != str: + func_name = str(func_name, 'utf-8') + cipher = getattr(libcrypto, func_name, None) + if cipher: + cipher.restype = c_void_p + return cipher() + return None + + +class CtypesCrypto(object): + def __init__(self, cipher_name, key, iv, op): + if not loaded: + load_openssl() + self._ctx = None + cipher = libcrypto.EVP_get_cipherbyname(cipher_name) + if not cipher: + cipher = load_cipher(cipher_name) + if not cipher: + raise Exception('cipher %s not found in libcrypto' % cipher_name) + key_ptr = c_char_p(key) + iv_ptr = c_char_p(iv) + self._ctx = libcrypto.EVP_CIPHER_CTX_new() + if not self._ctx: + raise Exception('can not create cipher context') + r = libcrypto.EVP_CipherInit_ex(self._ctx, cipher, None, + key_ptr, iv_ptr, c_int(op)) + if not r: + self.clean() + raise Exception('can not initialize cipher context') + + def update(self, data): + global buf_size, buf + cipher_out_len = c_long(0) + l = len(data) + if buf_size < l: + buf_size = l * 2 + buf = create_string_buffer(buf_size) + libcrypto.EVP_CipherUpdate(self._ctx, byref(buf), + byref(cipher_out_len), c_char_p(data), l) + # buf is copied to a str object when we access buf.raw + return buf.raw[:cipher_out_len.value] + + def __del__(self): + self.clean() + + def clean(self): + if self._ctx: + libcrypto.EVP_CIPHER_CTX_cleanup(self._ctx) + libcrypto.EVP_CIPHER_CTX_free(self._ctx) + + +ciphers = { + b'aes-128-cfb': (16, 16, CtypesCrypto), + b'aes-192-cfb': (24, 16, CtypesCrypto), + b'aes-256-cfb': (32, 16, CtypesCrypto), + b'aes-128-ofb': (16, 16, CtypesCrypto), + b'aes-192-ofb': (24, 16, CtypesCrypto), + b'aes-256-ofb': (32, 16, CtypesCrypto), + b'aes-128-ctr': (16, 16, CtypesCrypto), + b'aes-192-ctr': (24, 16, CtypesCrypto), + b'aes-256-ctr': (32, 16, CtypesCrypto), + b'aes-128-cfb8': (16, 16, CtypesCrypto), + b'aes-192-cfb8': (24, 16, CtypesCrypto), + b'aes-256-cfb8': (32, 16, CtypesCrypto), + b'aes-128-cfb1': (16, 16, CtypesCrypto), + b'aes-192-cfb1': (24, 16, CtypesCrypto), + b'aes-256-cfb1': (32, 16, CtypesCrypto), + b'bf-cfb': (16, 8, CtypesCrypto), + b'camellia-128-cfb': (16, 16, CtypesCrypto), + b'camellia-192-cfb': (24, 16, CtypesCrypto), + b'camellia-256-cfb': (32, 16, CtypesCrypto), + b'cast5-cfb': (16, 8, CtypesCrypto), + b'des-cfb': (8, 8, CtypesCrypto), + b'idea-cfb': (16, 8, CtypesCrypto), + b'rc2-cfb': (16, 8, CtypesCrypto), + b'rc4': (16, 0, CtypesCrypto), + b'seed-cfb': (16, 16, CtypesCrypto), +} + + +def run_method(method): + from shadowsocks.crypto import util + + cipher = CtypesCrypto(method, b'k' * 32, b'i' * 16, 1) + decipher = CtypesCrypto(method, b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + + +def test_aes_128_cfb(): + run_method(b'aes-128-cfb') + + +def test_aes_256_cfb(): + run_method(b'aes-256-cfb') + + +def test_aes_128_cfb8(): + run_method(b'aes-128-cfb8') + + +def test_aes_256_ofb(): + run_method(b'aes-256-ofb') + + +def test_aes_256_ctr(): + run_method(b'aes-256-ctr') + + +def test_bf_cfb(): + run_method(b'bf-cfb') + + +def test_rc4(): + run_method(b'rc4') + + +if __name__ == '__main__': + test_aes_128_cfb() diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py new file mode 100644 index 00000000..4c7e1480 --- /dev/null +++ b/shadowsocks/crypto/m2.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import sys +import logging + +__all__ = ['ciphers'] + +has_m2 = True +try: + __import__('M2Crypto') +except ImportError: + has_m2 = False + + +def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, + padding=1): + + import M2Crypto.EVP + return M2Crypto.EVP.Cipher(alg.replace('-', '_'), key, iv, op, + key_as_bytes=0, d='md5', salt=None, i=1, + padding=1) + + +def err(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): + logging.error(('M2Crypto is required to use %s, please run' + ' `apt-get install python-m2crypto`') % alg) + sys.exit(1) + + +if has_m2: + ciphers = { + b'aes-128-cfb': (16, 16, create_cipher), + b'aes-192-cfb': (24, 16, create_cipher), + b'aes-256-cfb': (32, 16, create_cipher), + b'bf-cfb': (16, 8, create_cipher), + b'camellia-128-cfb': (16, 16, create_cipher), + b'camellia-192-cfb': (24, 16, create_cipher), + b'camellia-256-cfb': (32, 16, create_cipher), + b'cast5-cfb': (16, 8, create_cipher), + b'des-cfb': (8, 8, create_cipher), + b'idea-cfb': (16, 8, create_cipher), + b'rc2-cfb': (16, 8, create_cipher), + b'rc4': (16, 0, create_cipher), + b'seed-cfb': (16, 16, create_cipher), + } +else: + ciphers = {} + + +def run_method(method): + from shadowsocks.crypto import util + + cipher = create_cipher(method, b'k' * 32, b'i' * 16, 1) + decipher = create_cipher(method, b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + + +def check_env(): + # skip this test on pypy and Python 3 + try: + import __pypy__ + del __pypy__ + from nose.plugins.skip import SkipTest + raise SkipTest + except ImportError: + pass + if bytes != str: + from nose.plugins.skip import SkipTest + raise SkipTest + + +def test_aes_128_cfb(): + check_env() + run_method(b'aes-128-cfb') + + +def test_aes_256_cfb(): + check_env() + run_method(b'aes-256-cfb') + + +def test_bf_cfb(): + check_env() + run_method(b'bf-cfb') + + +def test_rc4(): + check_env() + run_method(b'rc4') + + +if __name__ == '__main__': + test_aes_128_cfb() diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 4e87f415..834b18ce 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -47,6 +47,8 @@ def try_cipher(key, method=None): def EVP_BytesToKey(password, key_len, iv_len): # equivalent to OpenSSL's EVP_BytesToKey() with count 1 # so that we make the same key and iv as nodejs version + if hasattr(password, 'encode'): + password = password.encode('utf-8') cached_key = '%s-%d-%d' % (password, key_len, iv_len) r = cached_keys.get(cached_key, None) if r: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 4834883a..6b14114e 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -101,6 +101,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._loop = loop self._local_sock = local_sock self._remote_sock = None + self._remote_udp = False self._config = config self._dns_resolver = dns_resolver @@ -190,6 +191,28 @@ def _write_to_sock(self, data, sock): # and update the stream to wait for writing if not data or not sock: return False + #logging.debug("_write_to_sock %s %s %s" % (self._remote_sock, sock, self._remote_udp)) + if self._remote_sock == sock and self._remote_udp: + try: + addrtype = data[3] + if addrtype == '\x01': + addr = socket.inet_ntoa(data[4:8]) + port = struct.unpack('>H', data[8:10])[0] + logging.info('udp sendto %s:%d %d bytes from %s:%d' % (addr, port, len(data), self._client_address[0], self._client_address[1])) + sock.sendto(data[10:], (addr, port)) + elif addrtype == '\x04': + addr = socket.inet_ntop(data[4:20]) + port = struct.unpack('>H', data[20:22])[0] + logging.info('udp sendto %s:%d %d bytes from %s:%d' % (addr, port, len(data), self._client_address[0], self._client_address[1])) + sock.sendto(data[22:], (addr, port)) + elif addrtype == '\x03': + #unsupport + pass + except Exception as e: + trace = traceback.format_exc() + logging.error(trace) + return True + uncomplete = False try: l = len(data) @@ -203,6 +226,7 @@ def _write_to_sock(self, data, sock): errno.EWOULDBLOCK): uncomplete = True else: + #traceback.print_exc() shell.print_exception(e) self.destroy() return False @@ -291,11 +315,13 @@ def _handle_stage_addr(self, data): header_result = parse_header(data) if header_result is None: raise Exception('can not parse header') - addrtype, remote_addr, remote_port, header_length = header_result - logging.info('connecting %s:%d from %s:%d' % - (common.to_str(remote_addr), remote_port, - self._client_address[0], self._client_address[1])) + connecttype, remote_addr, remote_port, header_length = header_result + logging.info('%s connecting %s:%d from %s:%d' % + ((connecttype == 0) and 'tcp' or 'udp', + common.to_str(remote_addr), remote_port, + self._client_address[0], self._client_address[1])) self._remote_address = (common.to_str(remote_addr), remote_port) + self._remote_udp = (connecttype != 0) # pause reading self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) self._stage = STAGE_DNS @@ -323,8 +349,10 @@ def _handle_stage_addr(self, data): self.destroy() def _create_remote_socket(self, ip, port): - addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, - socket.SOL_TCP) + if self._remote_udp: + addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) + else: + addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) if len(addrs) == 0: raise Exception("getaddrinfo failed for %s:%d" % (ip, port)) af, socktype, proto, canonname, sa = addrs[0] @@ -336,7 +364,10 @@ def _create_remote_socket(self, ip, port): self._remote_sock = remote_sock self._fd_to_handlers[remote_sock.fileno()] = self remote_sock.setblocking(False) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + if self._remote_udp: + pass + else: + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) return remote_sock def _handle_dns_resolved(self, result, error): @@ -368,14 +399,18 @@ def _handle_dns_resolved(self, result, error): # else do connect remote_sock = self._create_remote_socket(remote_addr, remote_port) - try: - remote_sock.connect((remote_addr, remote_port)) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == \ - errno.EINPROGRESS: - pass - self._loop.add(remote_sock, - eventloop.POLL_ERR | eventloop.POLL_OUT) + if self._remote_udp: + self._loop.add(remote_sock, + eventloop.POLL_IN) + else: + try: + remote_sock.connect((remote_addr, remote_port)) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == \ + errno.EINPROGRESS: + pass + self._loop.add(remote_sock, + eventloop.POLL_ERR | eventloop.POLL_OUT) self._stage = STAGE_CONNECTING self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) @@ -407,6 +442,7 @@ def _on_local_read(self): data = self._encryptor.decrypt(data) if not data: return + self._server.server_transfer_ul += len(data) if self._stage == STAGE_STREAM: if self._is_local: data = self._encryptor.encrypt(data) @@ -428,14 +464,26 @@ def _on_remote_read(self): self._update_activity() data = None try: - data = self._remote_sock.recv(BUF_SIZE) + if self._remote_udp: + data, addr = self._remote_sock.recvfrom(BUF_SIZE) + port = struct.pack('>H', addr[1]) + try: + ip = socket.inet_aton(addr[0]) + data = '\x00\x00\x00\x01' + ip + port + data + except Exception as e: + ip = socket.inet_pton(socket.AF_INET6, addr[0]) + data = '\x00\x00\x00\x04' + ip + port + data + logging.info('udp recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) + else: + data = self._remote_sock.recv(BUF_SIZE) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) in \ - (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): + (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK, 10035): #errno.WSAEWOULDBLOCK return if not data: self.destroy() return + self._server.server_transfer_dl += len(data) if self._is_local: data = self._encryptor.decrypt(data) else: @@ -472,12 +520,16 @@ def _on_local_error(self): logging.debug('got local error') if self._local_sock: logging.error(eventloop.get_sock_error(self._local_sock)) + trace = traceback.format_exc() + logging.error(trace) self.destroy() def _on_remote_error(self): logging.debug('got remote error') if self._remote_sock: logging.error(eventloop.get_sock_error(self._remote_sock)) + trace = traceback.format_exc() + logging.error(trace) self.destroy() def handle_event(self, sock, event): @@ -558,6 +610,8 @@ def __init__(self, config, dns_resolver, is_local): self._eventloop = None self._fd_to_handlers = {} self._last_time = time.time() + self.server_transfer_ul = 0L + self.server_transfer_dl = 0L self._timeout = config['timeout'] self._timeouts = [] # a list for all the handlers diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 98bfaaa7..ba4e9d7b 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -173,6 +173,7 @@ def _handle_server(self): client = self._cache.get(key, None) if not client: # TODO async getaddrinfo + logging.info('UDP handle_server %s:%d from %s:%d' % (common.to_str(server_addr), server_port, self._listen_addr, self._listen_port)) addrs = socket.getaddrinfo(server_addr, server_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) if addrs: diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py new file mode 100644 index 00000000..785f18c6 --- /dev/null +++ b/shadowsocks/utils.py @@ -0,0 +1,278 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import json +import sys +import getopt +import logging + + +VERBOSE_LEVEL = 5 + + +def check_python(): + info = sys.version_info + if not (info[0] == 2 and info[1] >= 6): + print 'Python 2.6 or 2.7 required' + sys.exit(1) + + +def print_shadowsocks(): + version = '' + try: + import pkg_resources + version = pkg_resources.get_distribution('shadowsocks').version + except Exception: + pass + print 'shadowsocks %s' % version + + +def find_config(): + config_path = 'config.json' + if os.path.exists(config_path): + return config_path + config_path = os.path.join(os.path.dirname(__file__), '../', 'config.json') + if os.path.exists(config_path): + return config_path + return None + + +def check_config(config): + if config.get('local_address', '') in ['0.0.0.0']: + logging.warn('warning: local set to listen 0.0.0.0, which is not safe') + if config.get('server', '') in ['127.0.0.1', 'localhost']: + logging.warn('warning: server set to listen %s:%s, are you sure?' % + (config['server'], config['server_port'])) + if (config.get('method', '') or '').lower() == '': + logging.warn('warning: table is not safe; please use a safer cipher, ' + 'like AES-256-CFB') + if (config.get('method', '') or '').lower() == 'rc4': + logging.warn('warning: RC4 is not safe; please use a safer cipher, ' + 'like AES-256-CFB') + if config.get('timeout', 300) < 100: + logging.warn('warning: your timeout %d seems too short' % + int(config.get('timeout'))) + if config.get('timeout', 300) > 600: + logging.warn('warning: your timeout %d seems too long' % + int(config.get('timeout'))) + if config.get('password') in ['mypassword', 'barfoo!']: + logging.error('DON\'T USE DEFAULT PASSWORD! Please change it in your ' + 'config.json!') + exit(1) + + +def get_config(is_local): + logging.basicConfig(level=logging.INFO, + format='%(levelname)-s: %(message)s', filemode='a+') + if is_local: + shortopts = 'hs:b:p:k:l:m:c:t:vq' + longopts = ['fast-open'] + else: + shortopts = 'hs:p:k:m:c:t:vq' + longopts = ['fast-open', 'workers:'] + try: + config_path = find_config() + optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + for key, value in optlist: + if key == '-c': + config_path = value + + if config_path: + logging.info('loading config from %s' % config_path) + with open(config_path, 'rb') as f: + try: + config = json.load(f, object_hook=_decode_dict) + except ValueError as e: + logging.error('found an error in config.json: %s', + e.message) + sys.exit(1) + else: + config = {} + + optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + v_count = 0 + for key, value in optlist: + if key == '-p': + config['server_port'] = int(value) + elif key == '-k': + config['password'] = value + elif key == '-l': + config['local_port'] = int(value) + elif key == '-s': + config['server'] = value + elif key == '-m': + config['method'] = value + elif key == '-b': + config['local_address'] = value + elif key == '-v': + v_count += 1 + # '-vv' turns on more verbose mode + config['verbose'] = v_count + elif key == '-t': + config['timeout'] = int(value) + elif key == '--fast-open': + config['fast_open'] = True + elif key == '--workers': + config['workers'] = value + elif key == '-h': + if is_local: + print_local_help() + else: + print_server_help() + sys.exit(0) + elif key == '-q': + v_count -= 1 + config['verbose'] = v_count + except getopt.GetoptError as e: + print >>sys.stderr, e + print_help(is_local) + sys.exit(2) + + if not config: + logging.error('config not specified') + print_help(is_local) + sys.exit(2) + + config['password'] = config.get('password', None) + config['method'] = config.get('method', 'aes-256-cfb') + config['port_password'] = config.get('port_password', None) + config['timeout'] = int(config.get('timeout', 300)) + config['fast_open'] = config.get('fast_open', False) + config['workers'] = config.get('workers', 1) + config['verbose'] = config.get('verbose', False) + config['local_address'] = config.get('local_address', '127.0.0.1') + config['local_port'] = config.get('local_port', 1080) + if is_local: + if config.get('server', None) is None: + logging.error('server addr not specified') + print_local_help() + sys.exit(2) + else: + config['server'] = config.get('server', '0.0.0.0') + config['server_port'] = config.get('server_port', 8388) + + if not ('password' in config and config['password']): + logging.error('password not specified') + print_help(is_local) + sys.exit(2) + + logging.getLogger('').handlers = [] + logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE') + if config['verbose'] >= 2: + level = VERBOSE_LEVEL + elif config['verbose'] == 1: + level = logging.DEBUG + elif config['verbose'] == -1: + level = logging.WARN + elif config['verbose'] <= -2: + level = logging.ERROR + else: + level = logging.INFO + logging.basicConfig(level=level, + format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') + + check_config(config) + + return config + + +def print_help(is_local): + if is_local: + print_local_help() + else: + print_server_help() + + +def print_local_help(): + print '''usage: sslocal [-h] -s SERVER_ADDR [-p SERVER_PORT] + [-b LOCAL_ADDR] [-l LOCAL_PORT] -k PASSWORD [-m METHOD] + [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] [-q] + +optional arguments: + -h, --help show this help message and exit + -s SERVER_ADDR server address + -p SERVER_PORT server port, default: 8388 + -b LOCAL_ADDR local binding address, default: 127.0.0.1 + -l LOCAL_PORT local port, default: 1080 + -k PASSWORD password + -m METHOD encryption method, default: aes-256-cfb + -t TIMEOUT timeout in seconds, default: 300 + -c CONFIG path to config file + --fast-open use TCP_FASTOPEN, requires Linux 3.7+ + -v, -vv verbose mode + -q, -qq quiet mode, only show warnings/errors + +Online help: +''' + + +def print_server_help(): + print '''usage: ssserver [-h] [-s SERVER_ADDR] [-p SERVER_PORT] -k PASSWORD + -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] + [--workers WORKERS] [-v] [-q] + +optional arguments: + -h, --help show this help message and exit + -s SERVER_ADDR server address, default: 0.0.0.0 + -p SERVER_PORT server port, default: 8388 + -k PASSWORD password + -m METHOD encryption method, default: aes-256-cfb + -t TIMEOUT timeout in seconds, default: 300 + -c CONFIG path to config file + --fast-open use TCP_FASTOPEN, requires Linux 3.7+ + --workers WORKERS number of workers, available on Unix/Linux + -v, -vv verbose mode + -q, -qq quiet mode, only show warnings/errors + +Online help: +''' + + +def _decode_list(data): + rv = [] + for item in data: + if isinstance(item, unicode): + item = item.encode('utf-8') + elif isinstance(item, list): + item = _decode_list(item) + elif isinstance(item, dict): + item = _decode_dict(item) + rv.append(item) + return rv + + +def _decode_dict(data): + rv = {} + for key, value in data.iteritems(): + if isinstance(key, unicode): + key = key.encode('utf-8') + if isinstance(value, unicode): + value = value.encode('utf-8') + elif isinstance(value, list): + value = _decode_list(value) + elif isinstance(value, dict): + value = _decode_dict(value) + rv[key] = value + return rv diff --git a/switchrule.py b/switchrule.py new file mode 100644 index 00000000..ce6b76ea --- /dev/null +++ b/switchrule.py @@ -0,0 +1,2 @@ +def isTurnOn(plan, switch): + return True From c27f6283a26ed6a79540f6eac094d546379e4207 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 9 Jun 2015 23:43:45 +0800 Subject: [PATCH 002/487] UDP remote DNS resolve --- shadowsocks/common.py | 4 +- shadowsocks/tcprelay.py | 31 ++--- shadowsocks/udprelay.py | 4 +- shadowsocks/utils.py | 278 ---------------------------------------- 4 files changed, 17 insertions(+), 300 deletions(-) delete mode 100644 shadowsocks/utils.py diff --git a/shadowsocks/common.py b/shadowsocks/common.py index d740160b..cdd123cc 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -173,8 +173,8 @@ def parse_header(data): else: logging.warn('header is too short') else: - logging.warn('unsupported addrtype %d, maybe wrong password or ' - 'encryption method' % addrtype) + logging.warn('unsupported addrtype %d, maybe wrong password' % + addrtype) if dest_addr is None: return None return connecttype, to_bytes(dest_addr), dest_port, header_length diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 6b14114e..8d54c10c 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -194,20 +194,19 @@ def _write_to_sock(self, data, sock): #logging.debug("_write_to_sock %s %s %s" % (self._remote_sock, sock, self._remote_udp)) if self._remote_sock == sock and self._remote_udp: try: - addrtype = data[3] - if addrtype == '\x01': - addr = socket.inet_ntoa(data[4:8]) - port = struct.unpack('>H', data[8:10])[0] - logging.info('udp sendto %s:%d %d bytes from %s:%d' % (addr, port, len(data), self._client_address[0], self._client_address[1])) - sock.sendto(data[10:], (addr, port)) - elif addrtype == '\x04': - addr = socket.inet_ntop(data[4:20]) - port = struct.unpack('>H', data[20:22])[0] - logging.info('udp sendto %s:%d %d bytes from %s:%d' % (addr, port, len(data), self._client_address[0], self._client_address[1])) - sock.sendto(data[22:], (addr, port)) - elif addrtype == '\x03': - #unsupport - pass + #TODO + data = data[3:] + header_result = parse_header(data) + if header_result is None: + return False + connecttype, dest_addr, dest_port, header_length = header_result + addrs = socket.getaddrinfo(dest_addr, dest_port, 0, + socket.SOCK_DGRAM, socket.SOL_UDP) + if addrs: + af, socktype, proto, canonname, server_addr = addrs[0] + data = data[header_length:] + sock.sendto(data, server_addr) + except Exception as e: trace = traceback.format_exc() logging.error(trace) @@ -520,16 +519,12 @@ def _on_local_error(self): logging.debug('got local error') if self._local_sock: logging.error(eventloop.get_sock_error(self._local_sock)) - trace = traceback.format_exc() - logging.error(trace) self.destroy() def _on_remote_error(self): logging.debug('got remote error') if self._remote_sock: logging.error(eventloop.get_sock_error(self._remote_sock)) - trace = traceback.format_exc() - logging.error(trace) self.destroy() def handle_event(self, sock, event): diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index ba4e9d7b..71d7a95d 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -162,7 +162,7 @@ def _handle_server(self): header_result = parse_header(data) if header_result is None: return - addrtype, dest_addr, dest_port, header_length = header_result + connecttype, dest_addr, dest_port, header_length = header_result if self._is_local: server_addr, server_port = self._get_a_server() @@ -234,7 +234,7 @@ def _handle_client(self, sock): header_result = parse_header(data) if header_result is None: return - # addrtype, dest_addr, dest_port, header_length = header_result + # connecttype, dest_addr, dest_port, header_length = header_result response = b'\x00\x00\x00' + data client_addr = self._client_fd_to_server_addr.get(sock.fileno()) if client_addr: diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py deleted file mode 100644 index 785f18c6..00000000 --- a/shadowsocks/utils.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright (c) 2014 clowwindy -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import json -import sys -import getopt -import logging - - -VERBOSE_LEVEL = 5 - - -def check_python(): - info = sys.version_info - if not (info[0] == 2 and info[1] >= 6): - print 'Python 2.6 or 2.7 required' - sys.exit(1) - - -def print_shadowsocks(): - version = '' - try: - import pkg_resources - version = pkg_resources.get_distribution('shadowsocks').version - except Exception: - pass - print 'shadowsocks %s' % version - - -def find_config(): - config_path = 'config.json' - if os.path.exists(config_path): - return config_path - config_path = os.path.join(os.path.dirname(__file__), '../', 'config.json') - if os.path.exists(config_path): - return config_path - return None - - -def check_config(config): - if config.get('local_address', '') in ['0.0.0.0']: - logging.warn('warning: local set to listen 0.0.0.0, which is not safe') - if config.get('server', '') in ['127.0.0.1', 'localhost']: - logging.warn('warning: server set to listen %s:%s, are you sure?' % - (config['server'], config['server_port'])) - if (config.get('method', '') or '').lower() == '': - logging.warn('warning: table is not safe; please use a safer cipher, ' - 'like AES-256-CFB') - if (config.get('method', '') or '').lower() == 'rc4': - logging.warn('warning: RC4 is not safe; please use a safer cipher, ' - 'like AES-256-CFB') - if config.get('timeout', 300) < 100: - logging.warn('warning: your timeout %d seems too short' % - int(config.get('timeout'))) - if config.get('timeout', 300) > 600: - logging.warn('warning: your timeout %d seems too long' % - int(config.get('timeout'))) - if config.get('password') in ['mypassword', 'barfoo!']: - logging.error('DON\'T USE DEFAULT PASSWORD! Please change it in your ' - 'config.json!') - exit(1) - - -def get_config(is_local): - logging.basicConfig(level=logging.INFO, - format='%(levelname)-s: %(message)s', filemode='a+') - if is_local: - shortopts = 'hs:b:p:k:l:m:c:t:vq' - longopts = ['fast-open'] - else: - shortopts = 'hs:p:k:m:c:t:vq' - longopts = ['fast-open', 'workers:'] - try: - config_path = find_config() - optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) - for key, value in optlist: - if key == '-c': - config_path = value - - if config_path: - logging.info('loading config from %s' % config_path) - with open(config_path, 'rb') as f: - try: - config = json.load(f, object_hook=_decode_dict) - except ValueError as e: - logging.error('found an error in config.json: %s', - e.message) - sys.exit(1) - else: - config = {} - - optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) - v_count = 0 - for key, value in optlist: - if key == '-p': - config['server_port'] = int(value) - elif key == '-k': - config['password'] = value - elif key == '-l': - config['local_port'] = int(value) - elif key == '-s': - config['server'] = value - elif key == '-m': - config['method'] = value - elif key == '-b': - config['local_address'] = value - elif key == '-v': - v_count += 1 - # '-vv' turns on more verbose mode - config['verbose'] = v_count - elif key == '-t': - config['timeout'] = int(value) - elif key == '--fast-open': - config['fast_open'] = True - elif key == '--workers': - config['workers'] = value - elif key == '-h': - if is_local: - print_local_help() - else: - print_server_help() - sys.exit(0) - elif key == '-q': - v_count -= 1 - config['verbose'] = v_count - except getopt.GetoptError as e: - print >>sys.stderr, e - print_help(is_local) - sys.exit(2) - - if not config: - logging.error('config not specified') - print_help(is_local) - sys.exit(2) - - config['password'] = config.get('password', None) - config['method'] = config.get('method', 'aes-256-cfb') - config['port_password'] = config.get('port_password', None) - config['timeout'] = int(config.get('timeout', 300)) - config['fast_open'] = config.get('fast_open', False) - config['workers'] = config.get('workers', 1) - config['verbose'] = config.get('verbose', False) - config['local_address'] = config.get('local_address', '127.0.0.1') - config['local_port'] = config.get('local_port', 1080) - if is_local: - if config.get('server', None) is None: - logging.error('server addr not specified') - print_local_help() - sys.exit(2) - else: - config['server'] = config.get('server', '0.0.0.0') - config['server_port'] = config.get('server_port', 8388) - - if not ('password' in config and config['password']): - logging.error('password not specified') - print_help(is_local) - sys.exit(2) - - logging.getLogger('').handlers = [] - logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE') - if config['verbose'] >= 2: - level = VERBOSE_LEVEL - elif config['verbose'] == 1: - level = logging.DEBUG - elif config['verbose'] == -1: - level = logging.WARN - elif config['verbose'] <= -2: - level = logging.ERROR - else: - level = logging.INFO - logging.basicConfig(level=level, - format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') - - check_config(config) - - return config - - -def print_help(is_local): - if is_local: - print_local_help() - else: - print_server_help() - - -def print_local_help(): - print '''usage: sslocal [-h] -s SERVER_ADDR [-p SERVER_PORT] - [-b LOCAL_ADDR] [-l LOCAL_PORT] -k PASSWORD [-m METHOD] - [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] [-q] - -optional arguments: - -h, --help show this help message and exit - -s SERVER_ADDR server address - -p SERVER_PORT server port, default: 8388 - -b LOCAL_ADDR local binding address, default: 127.0.0.1 - -l LOCAL_PORT local port, default: 1080 - -k PASSWORD password - -m METHOD encryption method, default: aes-256-cfb - -t TIMEOUT timeout in seconds, default: 300 - -c CONFIG path to config file - --fast-open use TCP_FASTOPEN, requires Linux 3.7+ - -v, -vv verbose mode - -q, -qq quiet mode, only show warnings/errors - -Online help: -''' - - -def print_server_help(): - print '''usage: ssserver [-h] [-s SERVER_ADDR] [-p SERVER_PORT] -k PASSWORD - -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] - [--workers WORKERS] [-v] [-q] - -optional arguments: - -h, --help show this help message and exit - -s SERVER_ADDR server address, default: 0.0.0.0 - -p SERVER_PORT server port, default: 8388 - -k PASSWORD password - -m METHOD encryption method, default: aes-256-cfb - -t TIMEOUT timeout in seconds, default: 300 - -c CONFIG path to config file - --fast-open use TCP_FASTOPEN, requires Linux 3.7+ - --workers WORKERS number of workers, available on Unix/Linux - -v, -vv verbose mode - -q, -qq quiet mode, only show warnings/errors - -Online help: -''' - - -def _decode_list(data): - rv = [] - for item in data: - if isinstance(item, unicode): - item = item.encode('utf-8') - elif isinstance(item, list): - item = _decode_list(item) - elif isinstance(item, dict): - item = _decode_dict(item) - rv.append(item) - return rv - - -def _decode_dict(data): - rv = {} - for key, value in data.iteritems(): - if isinstance(key, unicode): - key = key.encode('utf-8') - if isinstance(value, unicode): - value = value.encode('utf-8') - elif isinstance(value, list): - value = _decode_list(value) - elif isinstance(value, dict): - value = _decode_dict(value) - rv[key] = value - return rv From 7c2fe9fd5615d1732f9871b0d7fa8579c9d59aad Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 10 Jun 2015 12:35:49 +0800 Subject: [PATCH 003/487] use db key --- db_transfer.py | 273 +++++++++++++++++++-------------------- server.py | 18 ++- server_pool.py | 340 ++++++++++++++++++++++++------------------------- switchrule.py | 3 +- 4 files changed, 317 insertions(+), 317 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 8126d0a3..fc1c80d4 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -7,142 +7,145 @@ import sys from server_pool import ServerPool import Config +import traceback class DbTransfer(object): - instance = None + instance = None + + def __init__(self): + self.last_get_transfer = {} + + @staticmethod + def get_instance(): + if DbTransfer.instance is None: + DbTransfer.instance = DbTransfer() + return DbTransfer.instance + + def push_db_all_user(self): + #更新用户流量到数据库 + last_transfer = self.last_get_transfer + curr_transfer = ServerPool.get_instance().get_servers_transfer() + #上次和本次的增量 + dt_transfer = {} + for id in curr_transfer.keys(): + if id in last_transfer: + if last_transfer[id][0] == curr_transfer[id][0] and last_transfer[id][1] == curr_transfer[id][1]: + continue + elif curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + continue + elif last_transfer[id][0] <= curr_transfer[id][0] and \ + last_transfer[id][1] <= curr_transfer[id][1]: + dt_transfer[id] = [curr_transfer[id][0] - last_transfer[id][0], + curr_transfer[id][1] - last_transfer[id][1]] + else: + dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + else: + if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + continue + dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + + self.last_get_transfer = curr_transfer + query_head = 'UPDATE user' + query_sub_when = '' + query_sub_when2 = '' + query_sub_in = None + last_time = time.time() + for id in dt_transfer.keys(): + query_sub_when += ' WHEN %s THEN u+%s' % (id, dt_transfer[id][0]) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, dt_transfer[id][1]) + if query_sub_in is not None: + query_sub_in += ',%s' % id + else: + query_sub_in = '%s' % id + if query_sub_when == '': + return + query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ + ' END, d = CASE port' + query_sub_when2 + \ + ' END, t = ' + str(int(last_time)) + \ + ' WHERE port IN (%s)' % query_sub_in + #print query_sql + conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, + passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') + cur = conn.cursor() + cur.execute(query_sql) + cur.close() + conn.commit() + conn.close() + + @staticmethod + def pull_db_all_user(): + #数据库所有用户信息 + keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'switch', 'enable', 'plan' ] + conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, + passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') + cur = conn.cursor() + cur.execute("SELECT " + ','.join(keys) + " FROM user") + rows = [] + for r in cur.fetchall(): + d = {} + for column in xrange(len(keys)): + d[keys[column]] = r[column] + rows.append(d) + cur.close() + conn.close() + return rows + + @staticmethod + def del_server_out_of_bound_safe(last_rows, rows): + #停止超流量的服务 + #启动没超流量的服务 + #需要动态载入switchrule，以便实时修改规则 + cur_servers = {} + for row in rows: + try: + import switchrule + allow = switchrule.isTurnOn(row) and row['enable'] == 1 and row['u'] + row['d'] < row['transfer_enable'] + except Exception, e: + allow = False + + port = row['port'] + passwd = row['passwd'] + cur_servers[port] = passwd + + if ServerPool.get_instance().server_is_run(port) > 0: + if not allow: + logging.info('db stop server at port [%s]' % (port,)) + ServerPool.get_instance().del_server(port) + elif (port in ServerPool.get_instance().tcp_servers_pool and ServerPool.get_instance().tcp_servers_pool[port]._config['password'] != passwd) \ + or (port in ServerPool.get_instance().tcp_ipv6_servers_pool and ServerPool.get_instance().tcp_ipv6_servers_pool[port]._config['password'] != passwd): + #password changed + logging.info('db stop server at port [%s] reason: password changed' % (port,)) + ServerPool.get_instance().del_server(port) + + if allow and ServerPool.get_instance().server_is_run(port) == 0: + logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) + ServerPool.get_instance().new_server(port, passwd) + + for row in last_rows: + if row['port'] in cur_servers: + pass + else: + logging.info('db stop server at port [%s] reason: port not exist' % (row['port'])) + ServerPool.get_instance().del_server(row['port']) + + @staticmethod + def thread_db(): + import socket + import time + timeout = 60 + socket.setdefaulttimeout(timeout) + last_rows = [] + while True: + try: + DbTransfer.get_instance().push_db_all_user() + rows = DbTransfer.get_instance().pull_db_all_user() + DbTransfer.del_server_out_of_bound_safe(last_rows, rows) + last_rows = rows + except Exception as e: + trace = traceback.format_exc() + logging.error(trace) + #logging.warn('db thread except:%s' % e) + finally: + time.sleep(15) - def __init__(self): - self.last_get_transfer = {} - - @staticmethod - def get_instance(): - if DbTransfer.instance is None: - DbTransfer.instance = DbTransfer() - return DbTransfer.instance - - def push_db_all_user(self): - #更新用户流量到数据库 - last_transfer = self.last_get_transfer - curr_transfer = ServerPool.get_instance().get_servers_transfer() - #上次和本次的增量 - dt_transfer = {} - for id in curr_transfer.keys(): - if id in last_transfer: - if last_transfer[id][0] == curr_transfer[id][0] and last_transfer[id][1] == curr_transfer[id][1]: - continue - elif curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: - continue - elif last_transfer[id][0] <= curr_transfer[id][0] and \ - last_transfer[id][1] <= curr_transfer[id][1]: - dt_transfer[id] = [curr_transfer[id][0] - last_transfer[id][0], - curr_transfer[id][1] - last_transfer[id][1]] - else: - dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] - else: - if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: - continue - dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] - - self.last_get_transfer = curr_transfer - query_head = 'UPDATE user' - query_sub_when = '' - query_sub_when2 = '' - query_sub_in = None - last_time = time.time() - for id in dt_transfer.keys(): - query_sub_when += ' WHEN %s THEN u+%s' % (id, dt_transfer[id][0]) - query_sub_when2 += ' WHEN %s THEN d+%s' % (id, dt_transfer[id][1]) - if query_sub_in is not None: - query_sub_in += ',%s' % id - else: - query_sub_in = '%s' % id - if query_sub_when == '': - return - query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ - ' END, d = CASE port' + query_sub_when2 + \ - ' END, t = ' + str(int(last_time)) + \ - ' WHERE port IN (%s)' % query_sub_in - #print query_sql - conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, - passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') - cur = conn.cursor() - cur.execute(query_sql) - cur.close() - conn.commit() - conn.close() - - @staticmethod - def pull_db_all_user(): - #数据库所有用户信息 - conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, - passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') - cur = conn.cursor() - cur.execute("SELECT port, u, d, transfer_enable, passwd, switch, enable, plan FROM user") - rows = [] - for r in cur.fetchall(): - rows.append(list(r)) - cur.close() - conn.close() - return rows - - @staticmethod - def del_server_out_of_bound_safe(last_rows, rows): - #停止超流量的服务 - #启动没超流量的服务 - #需要动态载入switchrule，以便实时修改规则 - cur_servers = {} - for row in rows: - try: - import switchrule - allow = switchrule.isTurnOn(row[7], row[5]) and row[6] == 1 and row[1] + row[2] < row[3] - except Exception, e: - allow = False - - cur_servers[row[0]] = row[4] - - if ServerPool.get_instance().server_is_run(row[0]) > 0: - if not allow: - logging.info('db stop server at port [%s]' % (row[0])) - ServerPool.get_instance().del_server(row[0]) - elif (row[0] in ServerPool.get_instance().tcp_servers_pool and ServerPool.get_instance().tcp_servers_pool[row[0]]._config['password'] != row[4]) \ - or (row[0] in ServerPool.get_instance().tcp_ipv6_servers_pool and ServerPool.get_instance().tcp_ipv6_servers_pool[row[0]]._config['password'] != row[4]): - #password changed - logging.info('db stop server at port [%s] reason: password changed' % (row[0])) - ServerPool.get_instance().del_server(row[0]) - elif ServerPool.get_instance().server_run_status(row[0]) is False: - if allow: - logging.info('db start server at port [%s] pass [%s]' % (row[0], row[4])) - ServerPool.get_instance().new_server(row[0], row[4]) - - for row in last_rows: - if row[0] in cur_servers: - if row[4] == cur_servers[row[0]]: - pass - else: - logging.info('db stop server at port [%s] reason: port not exist' % (row[0])) - ServerPool.get_instance().del_server(row[0]) - - @staticmethod - def thread_db(): - import socket - import time - timeout = 60 - socket.setdefaulttimeout(timeout) - last_rows = [] - while True: - #logging.warn('db loop') - - try: - DbTransfer.get_instance().push_db_all_user() - rows = DbTransfer.get_instance().pull_db_all_user() - DbTransfer.del_server_out_of_bound_safe(last_rows, rows) - last_rows = rows - except Exception as e: - logging.warn('db thread except:%s' % e) - finally: - time.sleep(15) - - -#SQLData.pull_db_all_user() -#print DbTransfer.get_instance().test() diff --git a/server.py b/server.py index ad1e5583..de03e474 100644 --- a/server.py +++ b/server.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - +#!/usr/bin/python import time import sys import thread @@ -11,12 +9,12 @@ import db_transfer #def test(): -# thread.start_new_thread(DbTransfer.thread_db, ()) -# Api.web_server() +# thread.start_new_thread(DbTransfer.thread_db, ()) +# Api.web_server() if __name__ == '__main__': - #server_pool.ServerPool.get_instance() - #server_pool.ServerPool.get_instance().new_server(2333, '2333') - thread.start_new_thread(db_transfer.DbTransfer.thread_db, ()) - while True: - time.sleep(99999) + #server_pool.ServerPool.get_instance() + #server_pool.ServerPool.get_instance().new_server(2333, '2333') + thread.start_new_thread(db_transfer.DbTransfer.thread_db, ()) + while True: + time.sleep(99999) diff --git a/server_pool.py b/server_pool.py index c1232715..857afdff 100644 --- a/server_pool.py +++ b/server_pool.py @@ -24,7 +24,7 @@ import os import logging import time -from shadowsocks import utils +from shadowsocks import shell from shadowsocks import eventloop from shadowsocks import tcprelay from shadowsocks import udprelay @@ -38,174 +38,172 @@ class ServerPool(object): - instance = None - - def __init__(self): - utils.check_python() - self.config = utils.get_config(False) - utils.print_shadowsocks() - self.dns_resolver = asyncdns.DNSResolver() - self.mgr = asyncmgr.ServerMgr() - self.udp_on = True ### UDP switch ===================================== - - self.tcp_servers_pool = {} - self.tcp_ipv6_servers_pool = {} - self.udp_servers_pool = {} - self.udp_ipv6_servers_pool = {} - - self.loop = eventloop.EventLoop() - thread.start_new_thread(ServerPool._loop, (self.loop, self.dns_resolver, self.mgr)) - - @staticmethod - def get_instance(): - if ServerPool.instance is None: - ServerPool.instance = ServerPool() - return ServerPool.instance - - @staticmethod - def _loop(loop, dns_resolver, mgr): - try: - mgr.add_to_loop(loop) - dns_resolver.add_to_loop(loop) - loop.run() - except (KeyboardInterrupt, IOError, OSError) as e: - logging.error(e) - import traceback - traceback.print_exc() - os.exit(0) - - def server_is_run(self, port): - port = int(port) - ret = 0 - if port in self.tcp_servers_pool: - ret = 1 - if port in self.tcp_ipv6_servers_pool: - ret |= 2 - return ret - - def server_run_status(self, port): - if 'server' in self.config: - if port not in self.tcp_servers_pool: - return False - if 'server_ipv6' in self.config: - if port not in self.tcp_ipv6_servers_pool: - return False - return True - - def new_server(self, port, password): - ret = True - port = int(port) - - if 'server_ipv6' in self.config: - if port in self.tcp_ipv6_servers_pool: - logging.info("server already at %s:%d" % (self.config['server_ipv6'], port)) - return 'this port server is already running' - else: - a_config = self.config.copy() - a_config['server'] = a_config['server_ipv6'] - a_config['server_port'] = port - a_config['password'] = password - try: - logging.info("starting server at %s:%d" % (a_config['server'], port)) - tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) - tcp_server.add_to_loop(self.loop) - self.tcp_ipv6_servers_pool.update({port: tcp_server}) - if self.udp_on: - udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) - udp_server.add_to_loop(self.loop) - self.udp_ipv6_servers_pool.update({port: udp_server}) - except Exception, e: - logging.warn("IPV6 exception") - logging.warn(e) - - if 'server' in self.config: - if port in self.tcp_servers_pool: - logging.info("server already at %s:%d" % (self.config['server'], port)) - return 'this port server is already running' - else: - a_config = self.config.copy() - a_config['server_port'] = port - a_config['password'] = password - try: - logging.info("starting server at %s:%d" % (a_config['server'], port)) - tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) - tcp_server.add_to_loop(self.loop) - self.tcp_servers_pool.update({port: tcp_server}) - if self.udp_on: - udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) - udp_server.add_to_loop(self.loop) - self.udp_servers_pool.update({port: udp_server}) - except Exception, e: - logging.warn("IPV4 exception") - logging.warn(e) - - return True - - def del_server(self, port): - port = int(port) - logging.info("del server at %d" % port) - try: - udpsock = socket(AF_INET, SOCK_DGRAM) - udpsock.sendto('%s:%s:0:0' % (Config.MANAGE_PASS, port), (Config.MANAGE_BIND_IP, Config.MANAGE_PORT)) - udpsock.close() - except Exception, e: - logging.warn(e) - return True - - def cb_del_server(self, port): - port = int(port) - - if port not in self.tcp_servers_pool: - logging.info("stopped server at %s:%d already stop" % (self.config['server'], port)) - else: - logging.info("stopped server at %s:%d" % (self.config['server'], port)) - try: - self.tcp_servers_pool[port].destroy() - del self.tcp_servers_pool[port] - except Exception, e: - logging.warn(e) - if self.udp_on: - try: - self.udp_servers_pool[port].destroy() - del self.udp_servers_pool[port] - except Exception, e: - logging.warn(e) - - if 'server_ipv6' in self.config: - if port not in self.tcp_ipv6_servers_pool: - logging.info("stopped server at %s:%d already stop" % (self.config['server_ipv6'], port)) - else: - logging.info("stopped server at %s:%d" % (self.config['server_ipv6'], port)) - try: - self.tcp_ipv6_servers_pool[port].destroy() - del self.tcp_ipv6_servers_pool[port] - except Exception, e: - logging.warn(e) - if self.udp_on: - try: - self.udp_ipv6_servers_pool[port].destroy() - del self.udp_ipv6_servers_pool[port] - except Exception, e: - logging.warn(e) - - return True - - def get_server_transfer(self, port): - port = int(port) - ret = [0, 0] - if port in self.tcp_servers_pool: - ret[0] = self.tcp_servers_pool[port].server_transfer_ul - ret[1] = self.tcp_servers_pool[port].server_transfer_dl - if port in self.tcp_ipv6_servers_pool: - ret[0] += self.tcp_ipv6_servers_pool[port].server_transfer_ul - ret[1] += self.tcp_ipv6_servers_pool[port].server_transfer_dl - return ret - - def get_servers_transfer(self): - servers = self.tcp_servers_pool.copy() - servers.update(self.tcp_ipv6_servers_pool) - ret = {} - for port in servers.keys(): - ret[port] = self.get_server_transfer(port) - return ret + instance = None + + def __init__(self): + shell.check_python() + self.config = shell.get_config(False) + shell.print_shadowsocks() + self.dns_resolver = asyncdns.DNSResolver() + self.mgr = asyncmgr.ServerMgr() + self.udp_on = True ### UDP switch ===================================== + + self.tcp_servers_pool = {} + self.tcp_ipv6_servers_pool = {} + self.udp_servers_pool = {} + self.udp_ipv6_servers_pool = {} + + self.loop = eventloop.EventLoop() + thread.start_new_thread(ServerPool._loop, (self.loop, self.dns_resolver, self.mgr)) + + @staticmethod + def get_instance(): + if ServerPool.instance is None: + ServerPool.instance = ServerPool() + return ServerPool.instance + + @staticmethod + def _loop(loop, dns_resolver, mgr): + try: + mgr.add_to_loop(loop) + dns_resolver.add_to_loop(loop) + loop.run() + except (KeyboardInterrupt, IOError, OSError) as e: + logging.error(e) + import traceback + traceback.print_exc() + os.exit(0) + + def server_is_run(self, port): + port = int(port) + ret = 0 + if port in self.tcp_servers_pool: + ret = 1 + if port in self.tcp_ipv6_servers_pool: + ret |= 2 + return ret + + def server_run_status(self, port): + if 'server' in self.config: + if port not in self.tcp_servers_pool: + return False + if 'server_ipv6' in self.config: + if port not in self.tcp_ipv6_servers_pool: + return False + return True + + def new_server(self, port, password): + ret = True + port = int(port) + + if 'server_ipv6' in self.config: + if port in self.tcp_ipv6_servers_pool: + logging.info("server already at %s:%d" % (self.config['server_ipv6'], port)) + return 'this port server is already running' + else: + a_config = self.config.copy() + a_config['server'] = a_config['server_ipv6'] + a_config['server_port'] = port + a_config['password'] = password + try: + logging.info("starting server at %s:%d" % (a_config['server'], port)) + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) + tcp_server.add_to_loop(self.loop) + self.tcp_ipv6_servers_pool.update({port: tcp_server}) + if self.udp_on: + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) + udp_server.add_to_loop(self.loop) + self.udp_ipv6_servers_pool.update({port: udp_server}) + except Exception, e: + logging.warn("IPV6 %s " % (e,)) + + if 'server' in self.config: + if port in self.tcp_servers_pool: + logging.info("server already at %s:%d" % (self.config['server'], port)) + return 'this port server is already running' + else: + a_config = self.config.copy() + a_config['server_port'] = port + a_config['password'] = password + try: + logging.info("starting server at %s:%d" % (a_config['server'], port)) + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) + tcp_server.add_to_loop(self.loop) + self.tcp_servers_pool.update({port: tcp_server}) + if self.udp_on: + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) + udp_server.add_to_loop(self.loop) + self.udp_servers_pool.update({port: udp_server}) + except Exception, e: + logging.warn("IPV4 %s " % (e,)) + + return True + + def del_server(self, port): + port = int(port) + logging.info("del server at %d" % port) + try: + udpsock = socket(AF_INET, SOCK_DGRAM) + udpsock.sendto('%s:%s:0:0' % (Config.MANAGE_PASS, port), (Config.MANAGE_BIND_IP, Config.MANAGE_PORT)) + udpsock.close() + except Exception, e: + logging.warn(e) + return True + + def cb_del_server(self, port): + port = int(port) + + if port not in self.tcp_servers_pool: + logging.info("stopped server at %s:%d already stop" % (self.config['server'], port)) + else: + logging.info("stopped server at %s:%d" % (self.config['server'], port)) + try: + self.tcp_servers_pool[port].destroy() + del self.tcp_servers_pool[port] + except Exception, e: + logging.warn(e) + if self.udp_on: + try: + self.udp_servers_pool[port].destroy() + del self.udp_servers_pool[port] + except Exception, e: + logging.warn(e) + + if 'server_ipv6' in self.config: + if port not in self.tcp_ipv6_servers_pool: + logging.info("stopped server at %s:%d already stop" % (self.config['server_ipv6'], port)) + else: + logging.info("stopped server at %s:%d" % (self.config['server_ipv6'], port)) + try: + self.tcp_ipv6_servers_pool[port].destroy() + del self.tcp_ipv6_servers_pool[port] + except Exception, e: + logging.warn(e) + if self.udp_on: + try: + self.udp_ipv6_servers_pool[port].destroy() + del self.udp_ipv6_servers_pool[port] + except Exception, e: + logging.warn(e) + + return True + + def get_server_transfer(self, port): + port = int(port) + ret = [0, 0] + if port in self.tcp_servers_pool: + ret[0] = self.tcp_servers_pool[port].server_transfer_ul + ret[1] = self.tcp_servers_pool[port].server_transfer_dl + if port in self.tcp_ipv6_servers_pool: + ret[0] += self.tcp_ipv6_servers_pool[port].server_transfer_ul + ret[1] += self.tcp_ipv6_servers_pool[port].server_transfer_dl + return ret + + def get_servers_transfer(self): + servers = self.tcp_servers_pool.copy() + servers.update(self.tcp_ipv6_servers_pool) + ret = {} + for port in servers.keys(): + ret[port] = self.get_server_transfer(port) + return ret diff --git a/switchrule.py b/switchrule.py index ce6b76ea..2a0b324f 100644 --- a/switchrule.py +++ b/switchrule.py @@ -1,2 +1,3 @@ -def isTurnOn(plan, switch): +def isTurnOn(row): return True + From 1e71fba852bba2203fb7f6f4640d380c73303801 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 10 Jun 2015 14:21:22 +0800 Subject: [PATCH 004/487] drop UDP message if frag != 0 --- shadowsocks/tcprelay.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8d54c10c..18aa1f53 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -194,8 +194,12 @@ def _write_to_sock(self, data, sock): #logging.debug("_write_to_sock %s %s %s" % (self._remote_sock, sock, self._remote_udp)) if self._remote_sock == sock and self._remote_udp: try: - #TODO - data = data[3:] + frag = common.ord(data[2]) + if frag != 0: + logging.warn('drop a message since frag is %d' % (frag,)) + return False + else: + data = data[3:] header_result = parse_header(data) if header_result is None: return False From 616a0c74bd9e95cbd73230a0756f251f57f7bb22 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Thu, 11 Jun 2015 00:41:51 +0800 Subject: [PATCH 005/487] rename destroy to close --- server_pool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server_pool.py b/server_pool.py index 857afdff..0f0052ed 100644 --- a/server_pool.py +++ b/server_pool.py @@ -158,13 +158,13 @@ def cb_del_server(self, port): else: logging.info("stopped server at %s:%d" % (self.config['server'], port)) try: - self.tcp_servers_pool[port].destroy() + self.tcp_servers_pool[port].close() del self.tcp_servers_pool[port] except Exception, e: logging.warn(e) if self.udp_on: try: - self.udp_servers_pool[port].destroy() + self.udp_servers_pool[port].close() del self.udp_servers_pool[port] except Exception, e: logging.warn(e) @@ -175,13 +175,13 @@ def cb_del_server(self, port): else: logging.info("stopped server at %s:%d" % (self.config['server_ipv6'], port)) try: - self.tcp_ipv6_servers_pool[port].destroy() + self.tcp_ipv6_servers_pool[port].close() del self.tcp_ipv6_servers_pool[port] except Exception, e: logging.warn(e) if self.udp_on: try: - self.udp_ipv6_servers_pool[port].destroy() + self.udp_ipv6_servers_pool[port].close() del self.udp_ipv6_servers_pool[port] except Exception, e: logging.warn(e) From 565812ae1531937f46d94c34cf53c5baed66919a Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Thu, 11 Jun 2015 01:07:54 +0800 Subject: [PATCH 006/487] elaborate reasons of header parsing failure --- shadowsocks/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index cdd123cc..d740160b 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -173,8 +173,8 @@ def parse_header(data): else: logging.warn('header is too short') else: - logging.warn('unsupported addrtype %d, maybe wrong password' % - addrtype) + logging.warn('unsupported addrtype %d, maybe wrong password or ' + 'encryption method' % addrtype) if dest_addr is None: return None return connecttype, to_bytes(dest_addr), dest_port, header_length From 9b7dab692b4c89ddd6a0ff8d566f5d3c8b808092 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Thu, 11 Jun 2015 10:36:39 +0800 Subject: [PATCH 007/487] skip bind ipv4 addr if bind ipv6 success fix close server --- server_pool.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/server_pool.py b/server_pool.py index 0f0052ed..c75655a2 100644 --- a/server_pool.py +++ b/server_pool.py @@ -95,6 +95,7 @@ def server_run_status(self, port): def new_server(self, port, password): ret = True port = int(port) + ipv6_ok = False if 'server_ipv6' in self.config: if port in self.tcp_ipv6_servers_pool: @@ -102,11 +103,13 @@ def new_server(self, port, password): return 'this port server is already running' else: a_config = self.config.copy() + if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": + a_config['server_ipv6'] = a_config['server_ipv6'][1:-1] a_config['server'] = a_config['server_ipv6'] a_config['server_port'] = port a_config['password'] = password try: - logging.info("starting server at %s:%d" % (a_config['server'], port)) + logging.info("starting server at [%s]:%d" % (a_config['server'], port)) tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) tcp_server.add_to_loop(self.loop) self.tcp_ipv6_servers_pool.update({port: tcp_server}) @@ -114,10 +117,12 @@ def new_server(self, port, password): udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) udp_server.add_to_loop(self.loop) self.udp_ipv6_servers_pool.update({port: udp_server}) + if a_config['server_ipv6'] == "::": + ipv6_ok = True except Exception, e: logging.warn("IPV6 %s " % (e,)) - if 'server' in self.config: + if not ipv6_ok and 'server' in self.config: if port in self.tcp_servers_pool: logging.info("server already at %s:%d" % (self.config['server'], port)) return 'this port server is already running' @@ -158,13 +163,13 @@ def cb_del_server(self, port): else: logging.info("stopped server at %s:%d" % (self.config['server'], port)) try: - self.tcp_servers_pool[port].close() + self.tcp_servers_pool[port].close(True) del self.tcp_servers_pool[port] except Exception, e: logging.warn(e) if self.udp_on: try: - self.udp_servers_pool[port].close() + self.udp_servers_pool[port].close(True) del self.udp_servers_pool[port] except Exception, e: logging.warn(e) @@ -175,13 +180,13 @@ def cb_del_server(self, port): else: logging.info("stopped server at %s:%d" % (self.config['server_ipv6'], port)) try: - self.tcp_ipv6_servers_pool[port].close() + self.tcp_ipv6_servers_pool[port].close(True) del self.tcp_ipv6_servers_pool[port] except Exception, e: logging.warn(e) if self.udp_on: try: - self.udp_ipv6_servers_pool[port].close() + self.udp_ipv6_servers_pool[port].close(True) del self.udp_ipv6_servers_pool[port] except Exception, e: logging.warn(e) From 99db626225508c205f6b8e5982224cd9e28443e7 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Mon, 15 Jun 2015 12:00:00 +0800 Subject: [PATCH 008/487] run script fix server restart --- db_transfer.py | 2 +- run.sh | 6 ++++++ stop.sh | 3 +++ tail.sh | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 run.sh create mode 100644 stop.sh create mode 100644 tail.sh diff --git a/db_transfer.py b/db_transfer.py index fc1c80d4..3e95f5c3 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -118,7 +118,7 @@ def del_server_out_of_bound_safe(last_rows, rows): logging.info('db stop server at port [%s] reason: password changed' % (port,)) ServerPool.get_instance().del_server(port) - if allow and ServerPool.get_instance().server_is_run(port) == 0: + elif allow and ServerPool.get_instance().server_run_status(port) is False: logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) ServerPool.get_instance().new_server(port, passwd) diff --git a/run.sh b/run.sh new file mode 100644 index 00000000..a4698529 --- /dev/null +++ b/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +eval $(ps -ef | grep "[0-9] python server\\.py" | awk '{print "kill "$2}') +nohup python server.py >> ssserver.log 2>&1 & + +tail -f ssserver.log diff --git a/stop.sh b/stop.sh new file mode 100644 index 00000000..2fa55cf9 --- /dev/null +++ b/stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +eval $(ps -ef | grep "[0-9] python server\\.py" | awk '{print "kill "$2}') diff --git a/tail.sh b/tail.sh new file mode 100644 index 00000000..aa371393 --- /dev/null +++ b/tail.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +tail -f ssserver.log From 07dc98f60d7d09f3cfff7a21a67b268f96ce49ad Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 16 Jun 2015 13:00:30 +0800 Subject: [PATCH 009/487] udp fix sendto argv --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 18aa1f53..070e234a 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -209,7 +209,7 @@ def _write_to_sock(self, data, sock): if addrs: af, socktype, proto, canonname, server_addr = addrs[0] data = data[header_length:] - sock.sendto(data, server_addr) + sock.sendto(data, (server_addr[0], dest_port)) except Exception as e: trace = traceback.format_exc() From 4b52f833769ae9240c959bb70d5772ef742e66af Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 16 Jun 2015 13:10:43 +0800 Subject: [PATCH 010/487] udp ipv6 --- shadowsocks/tcprelay.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 070e234a..66fef68a 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -352,8 +352,14 @@ def _handle_stage_addr(self, data): self.destroy() def _create_remote_socket(self, ip, port): + addrs = None if self._remote_udp: - addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) + try: + addrs = socket.getaddrinfo("::", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) + except Exception as e: + pass + if addrs is None: + addrs = socket.getaddrinfo("0.0.0.0", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) else: addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) if len(addrs) == 0: From 17122c299c24a22f56c093139264865e7faa5982 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 16 Jun 2015 21:55:36 +0800 Subject: [PATCH 011/487] fix udp ipv6 --- shadowsocks/tcprelay.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 66fef68a..5fa9d8b4 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -351,14 +351,18 @@ def _handle_stage_addr(self, data): # TODO use logging when debug completed self.destroy() + def _is_support_ipv6(self): + local = socket.gethostbyaddr(socket.gethostname()) + for ip in local: + if ':' in ip: + return True + return False + def _create_remote_socket(self, ip, port): - addrs = None if self._remote_udp: - try: + if self._is_support_ipv6(): addrs = socket.getaddrinfo("::", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) - except Exception as e: - pass - if addrs is None: + else: addrs = socket.getaddrinfo("0.0.0.0", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) else: addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) From 1f530ff8f103bf21d548928274ab534f12bc28ac Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Sat, 20 Jun 2015 17:56:33 +0800 Subject: [PATCH 012/487] fix udp ipv6 --- shadowsocks/tcprelay.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 5fa9d8b4..d86c9885 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -94,6 +94,8 @@ class TCPRelayHandler(object): + support_ipv6 = None + def __init__(self, server, fd_to_handlers, loop, local_sock, config, dns_resolver, is_local): self._server = server @@ -351,13 +353,21 @@ def _handle_stage_addr(self, data): # TODO use logging when debug completed self.destroy() - def _is_support_ipv6(self): - local = socket.gethostbyaddr(socket.gethostname()) - for ip in local: - if ':' in ip: + def _has_ipv6_addr(self, addr_list): + for item in addr_list: + if type(item) is list: + if self._has_ipv6_addr(item): + return True + elif ':' in item: return True return False + def _is_support_ipv6(self): + if TCPRelayHandler.support_ipv6 is None: + local = socket.gethostbyaddr(socket.gethostname()) + TCPRelayHandler.support_ipv6 = self._has_ipv6_addr(local) + return TCPRelayHandler.support_ipv6 + def _create_remote_socket(self, ip, port): if self._remote_udp: if self._is_support_ipv6(): From 28318b5c1a52c94e0c8f72c4e4cce254670a7bef Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Sat, 20 Jun 2015 18:59:20 +0800 Subject: [PATCH 013/487] double UDP sockets for IPv4 and IPv6 --- shadowsocks/tcprelay.py | 55 ++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index d86c9885..729aa046 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -94,8 +94,6 @@ class TCPRelayHandler(object): - support_ipv6 = None - def __init__(self, server, fd_to_handlers, loop, local_sock, config, dns_resolver, is_local): self._server = server @@ -103,6 +101,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._loop = loop self._local_sock = local_sock self._remote_sock = None + self._remote_sock_v6 = None self._remote_udp = False self._config = config self._dns_resolver = dns_resolver @@ -194,7 +193,7 @@ def _write_to_sock(self, data, sock): if not data or not sock: return False #logging.debug("_write_to_sock %s %s %s" % (self._remote_sock, sock, self._remote_udp)) - if self._remote_sock == sock and self._remote_udp: + if self._remote_udp and self._remote_sock == sock: try: frag = common.ord(data[2]) if frag != 0: @@ -211,11 +210,15 @@ def _write_to_sock(self, data, sock): if addrs: af, socktype, proto, canonname, server_addr = addrs[0] data = data[header_length:] - sock.sendto(data, (server_addr[0], dest_port)) + if af == socket.AF_INET6: + self._remote_sock_v6.sendto(data, (server_addr[0], dest_port)) + else: + sock.sendto(data, (server_addr[0], dest_port)) except Exception as e: - trace = traceback.format_exc() - logging.error(trace) + #trace = traceback.format_exc() + #logging.error(trace) + logging.error(e) return True uncomplete = False @@ -362,18 +365,10 @@ def _has_ipv6_addr(self, addr_list): return True return False - def _is_support_ipv6(self): - if TCPRelayHandler.support_ipv6 is None: - local = socket.gethostbyaddr(socket.gethostname()) - TCPRelayHandler.support_ipv6 = self._has_ipv6_addr(local) - return TCPRelayHandler.support_ipv6 - def _create_remote_socket(self, ip, port): if self._remote_udp: - if self._is_support_ipv6(): - addrs = socket.getaddrinfo("::", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) - else: - addrs = socket.getaddrinfo("0.0.0.0", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) + addrs_v6 = socket.getaddrinfo("::", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) + addrs = socket.getaddrinfo("0.0.0.0", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) else: addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) if len(addrs) == 0: @@ -385,7 +380,15 @@ def _create_remote_socket(self, ip, port): common.to_str(sa[0])) remote_sock = socket.socket(af, socktype, proto) self._remote_sock = remote_sock + self._fd_to_handlers[remote_sock.fileno()] = self + + if self._remote_udp: + af, socktype, proto, canonname, sa = addrs_v6[0] + remote_sock_v6 = socket.socket(af, socktype, proto) + self._remote_sock_v6 = remote_sock_v6 + self._fd_to_handlers[remote_sock_v6.fileno()] = self + remote_sock.setblocking(False) if self._remote_udp: pass @@ -425,6 +428,9 @@ def _handle_dns_resolved(self, result, error): if self._remote_udp: self._loop.add(remote_sock, eventloop.POLL_IN) + if self._remote_sock_v6: + self._loop.add(self._remote_sock_v6, + eventloop.POLL_IN) else: try: remote_sock.connect((remote_addr, remote_port)) @@ -482,13 +488,16 @@ def _on_local_read(self): (not is_local and self._stage == STAGE_INIT): self._handle_stage_addr(data) - def _on_remote_read(self): + def _on_remote_read(self, is_remote_sock): # handle all remote read events self._update_activity() data = None try: if self._remote_udp: - data, addr = self._remote_sock.recvfrom(BUF_SIZE) + if is_remote_sock: + data, addr = self._remote_sock.recvfrom(BUF_SIZE) + else: + data, addr = self._remote_sock_v6.recvfrom(BUF_SIZE) port = struct.pack('>H', addr[1]) try: ip = socket.inet_aton(addr[0]) @@ -557,13 +566,13 @@ def handle_event(self, sock, event): logging.debug('ignore handle_event: destroyed') return # order is important - if sock == self._remote_sock: + if sock == self._remote_sock or sock == self._remote_sock_v6: if event & eventloop.POLL_ERR: self._on_remote_error() if self._stage == STAGE_DESTROYED: return if event & (eventloop.POLL_IN | eventloop.POLL_HUP): - self._on_remote_read() + self._on_remote_read(sock == self._remote_sock) if self._stage == STAGE_DESTROYED: return if event & eventloop.POLL_OUT: @@ -610,6 +619,12 @@ def destroy(self): del self._fd_to_handlers[self._remote_sock.fileno()] self._remote_sock.close() self._remote_sock = None + if self._remote_sock_v6: + logging.debug('destroying remote') + self._loop.remove(self._remote_sock_v6) + del self._fd_to_handlers[self._remote_sock_v6.fileno()] + self._remote_sock_v6.close() + self._remote_sock_v6 = None if self._local_sock: logging.debug('destroying local') self._loop.remove(self._local_sock) From d4ca0d281c1e0dec4ae13b56ea358709dc0b8722 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Mon, 22 Jun 2015 19:02:46 +0800 Subject: [PATCH 014/487] always UDP on random length TCP/UDP packet --- server_pool.py | 46 +++++++++++++++++++++-------------------- shadowsocks/common.py | 14 +++++++++++++ shadowsocks/tcprelay.py | 18 ++++++---------- shadowsocks/udprelay.py | 8 +++++-- 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/server_pool.py b/server_pool.py index c75655a2..566154c4 100644 --- a/server_pool.py +++ b/server_pool.py @@ -46,7 +46,6 @@ def __init__(self): shell.print_shadowsocks() self.dns_resolver = asyncdns.DNSResolver() self.mgr = asyncmgr.ServerMgr() - self.udp_on = True ### UDP switch ===================================== self.tcp_servers_pool = {} self.tcp_ipv6_servers_pool = {} @@ -110,19 +109,21 @@ def new_server(self, port, password): a_config['password'] = password try: logging.info("starting server at [%s]:%d" % (a_config['server'], port)) + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) tcp_server.add_to_loop(self.loop) self.tcp_ipv6_servers_pool.update({port: tcp_server}) - if self.udp_on: - udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) - udp_server.add_to_loop(self.loop) - self.udp_ipv6_servers_pool.update({port: udp_server}) + + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) + udp_server.add_to_loop(self.loop) + self.udp_ipv6_servers_pool.update({port: udp_server}) + if a_config['server_ipv6'] == "::": ipv6_ok = True except Exception, e: logging.warn("IPV6 %s " % (e,)) - if not ipv6_ok and 'server' in self.config: + if 'server' in self.config: if port in self.tcp_servers_pool: logging.info("server already at %s:%d" % (self.config['server'], port)) return 'this port server is already running' @@ -132,13 +133,16 @@ def new_server(self, port, password): a_config['password'] = password try: logging.info("starting server at %s:%d" % (a_config['server'], port)) - tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) - tcp_server.add_to_loop(self.loop) - self.tcp_servers_pool.update({port: tcp_server}) - if self.udp_on: + + if not ipv6_ok: + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) + tcp_server.add_to_loop(self.loop) + self.tcp_servers_pool.update({port: tcp_server}) + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) udp_server.add_to_loop(self.loop) self.udp_servers_pool.update({port: udp_server}) + except Exception, e: logging.warn("IPV4 %s " % (e,)) @@ -167,12 +171,11 @@ def cb_del_server(self, port): del self.tcp_servers_pool[port] except Exception, e: logging.warn(e) - if self.udp_on: - try: - self.udp_servers_pool[port].close(True) - del self.udp_servers_pool[port] - except Exception, e: - logging.warn(e) + try: + self.udp_servers_pool[port].close(True) + del self.udp_servers_pool[port] + except Exception, e: + logging.warn(e) if 'server_ipv6' in self.config: if port not in self.tcp_ipv6_servers_pool: @@ -184,12 +187,11 @@ def cb_del_server(self, port): del self.tcp_ipv6_servers_pool[port] except Exception, e: logging.warn(e) - if self.udp_on: - try: - self.udp_ipv6_servers_pool[port].close(True) - del self.udp_ipv6_servers_pool[port] - except Exception, e: - logging.warn(e) + try: + self.udp_ipv6_servers_pool[port].close(True) + del self.udp_ipv6_servers_pool[port] + except Exception, e: + logging.warn(e) return True diff --git a/shadowsocks/common.py b/shadowsocks/common.py index d740160b..605fbfac 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -138,6 +138,20 @@ def pack_addr(address): address = address[:255] # TODO return b'\x03' + chr(len(address)) + address +def pre_parse_header(data): + datatype = ord(data[0]) + if datatype == 0x80 : + if len(data) <= 2: + return None + rand_data_size = ord(data[1]) + if rand_data_size + 2 >= len(data): + logging.warn('header too short, maybe wrong password or ' + 'encryption method') + return None + data = data[rand_data_size + 2:] + elif datatype == 0x81: + data = data[1:] + return data def parse_header(data): addrtype = ord(data[0]) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 729aa046..6bb6bbc9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -27,7 +27,7 @@ import random from shadowsocks import encrypt, eventloop, shell, common -from shadowsocks.common import parse_header +from shadowsocks.common import pre_parse_header, parse_header # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 @@ -320,12 +320,15 @@ def _handle_stage_addr(self, data): logging.error('unknown command %d', cmd) self.destroy() return + data = pre_parse_header(data) + if data is None: + raise Exception('can not parse header') header_result = parse_header(data) if header_result is None: raise Exception('can not parse header') connecttype, remote_addr, remote_port, header_length = header_result logging.info('%s connecting %s:%d from %s:%d' % - ((connecttype == 0) and 'tcp' or 'udp', + ((connecttype == 0) and 'TCP' or 'UDP', common.to_str(remote_addr), remote_port, self._client_address[0], self._client_address[1])) self._remote_address = (common.to_str(remote_addr), remote_port) @@ -356,15 +359,6 @@ def _handle_stage_addr(self, data): # TODO use logging when debug completed self.destroy() - def _has_ipv6_addr(self, addr_list): - for item in addr_list: - if type(item) is list: - if self._has_ipv6_addr(item): - return True - elif ':' in item: - return True - return False - def _create_remote_socket(self, ip, port): if self._remote_udp: addrs_v6 = socket.getaddrinfo("::", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) @@ -505,7 +499,7 @@ def _on_remote_read(self, is_remote_sock): except Exception as e: ip = socket.inet_pton(socket.AF_INET6, addr[0]) data = '\x00\x00\x00\x04' + ip + port + data - logging.info('udp recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) + logging.info('UDP recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) else: data = self._remote_sock.recv(BUF_SIZE) except (OSError, IOError) as e: diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 71d7a95d..09f077fd 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -70,7 +70,7 @@ import random from shadowsocks import encrypt, eventloop, lru_cache, common, shell -from shadowsocks.common import parse_header, pack_addr +from shadowsocks.common import pre_parse_header, parse_header, pack_addr BUF_SIZE = 65536 @@ -159,6 +159,10 @@ def _handle_server(self): if not data: logging.debug('UDP handle_server: data is empty after decrypt') return + data = pre_parse_header(data) + if data is None: + return + header_result = parse_header(data) if header_result is None: return @@ -173,7 +177,7 @@ def _handle_server(self): client = self._cache.get(key, None) if not client: # TODO async getaddrinfo - logging.info('UDP handle_server %s:%d from %s:%d' % (common.to_str(server_addr), server_port, self._listen_addr, self._listen_port)) + #logging.info('UDP handle_server %s:%d from %s:%d' % (common.to_str(server_addr), server_port, self._listen_addr, self._listen_port)) addrs = socket.getaddrinfo(server_addr, server_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) if addrs: From ad588af75f517d51c716854b6a772c130ddd9086 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 24 Jun 2015 12:56:46 +0800 Subject: [PATCH 015/487] single user script --- shadowsocks/run.sh | 6 ++++++ shadowsocks/server.py | 34 ++++++++++++++++++++++++++++------ shadowsocks/stop.sh | 3 +++ shadowsocks/tail.sh | 3 +++ 4 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 shadowsocks/run.sh create mode 100644 shadowsocks/stop.sh create mode 100644 shadowsocks/tail.sh diff --git a/shadowsocks/run.sh b/shadowsocks/run.sh new file mode 100644 index 00000000..d1aa82c4 --- /dev/null +++ b/shadowsocks/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') +nohup python server.py a >> ssserver.log 2>&1 & + +tail -f ssserver.log diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 429a20a3..bcccdb5c 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -53,12 +53,34 @@ def main(): dns_resolver = asyncdns.DNSResolver() for port, password in config['port_password'].items(): a_config = config.copy() - a_config['server_port'] = int(port) - a_config['password'] = password - logging.info("starting server at %s:%d" % - (a_config['server'], int(port))) - tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) - udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) + ipv6_ok = False + if 'server_ipv6' in a_config: + try: + if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": + a_config['server_ipv6'] = a_config['server_ipv6'][1:-1] + a_config['server_port'] = int(port) + a_config['password'] = password + a_config['server'] = a_config['server_ipv6'] + logging.info("starting server at %s:%d" % + (a_config['server'], int(port))) + tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) + udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) + if a_config['server_ipv6'] == "::": + ipv6_ok = True + except Exception as e: + shell.print_exception(e) + + if not ipv6_ok: + try: + a_config = config.copy() + a_config['server_port'] = int(port) + a_config['password'] = password + logging.info("starting server at %s:%d" % + (a_config['server'], int(port))) + tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) + udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) + except Exception as e: + shell.print_exception(e) def run_server(): def child_handler(signum, _): diff --git a/shadowsocks/stop.sh b/shadowsocks/stop.sh new file mode 100644 index 00000000..af1fbf92 --- /dev/null +++ b/shadowsocks/stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') diff --git a/shadowsocks/tail.sh b/shadowsocks/tail.sh new file mode 100644 index 00000000..aa371393 --- /dev/null +++ b/shadowsocks/tail.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +tail -f ssserver.log From 543a5903a1d39d8d082507b79015b136f0f39407 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 26 Jun 2015 11:33:59 +0800 Subject: [PATCH 016/487] encrypt speed test --- server_pool.py | 16 ++++++++-------- shadowsocks/encrypt_test.py | 33 +++++++++++++++++++++++++++++++++ shadowsocks/server.py | 21 +++++++++++---------- 3 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 shadowsocks/encrypt_test.py diff --git a/server_pool.py b/server_pool.py index 566154c4..cdeba36a 100644 --- a/server_pool.py +++ b/server_pool.py @@ -134,17 +134,17 @@ def new_server(self, port, password): try: logging.info("starting server at %s:%d" % (a_config['server'], port)) - if not ipv6_ok: - tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) - tcp_server.add_to_loop(self.loop) - self.tcp_servers_pool.update({port: tcp_server}) + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) + tcp_server.add_to_loop(self.loop) + self.tcp_servers_pool.update({port: tcp_server}) - udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) - udp_server.add_to_loop(self.loop) - self.udp_servers_pool.update({port: udp_server}) + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) + udp_server.add_to_loop(self.loop) + self.udp_servers_pool.update({port: udp_server}) except Exception, e: - logging.warn("IPV4 %s " % (e,)) + if not ipv6_ok: + logging.warn("IPV4 %s " % (e,)) return True diff --git a/shadowsocks/encrypt_test.py b/shadowsocks/encrypt_test.py new file mode 100644 index 00000000..68228e18 --- /dev/null +++ b/shadowsocks/encrypt_test.py @@ -0,0 +1,33 @@ +from __future__ import absolute_import, division, print_function, \ + with_statement + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) + + +from crypto import rc4_md5 +from crypto import openssl +from crypto import sodium +from crypto import table + +def main(): + print("\n""rc4_md5") + rc4_md5.test() + print("\n""aes-256-cfb") + openssl.test_aes_256_cfb() + print("\n""aes-128-cfb") + openssl.test_aes_128_cfb() + print("\n""rc4") + openssl.test_rc4() + print("\n""salsa20") + sodium.test_salsa20() + print("\n""chacha20") + sodium.test_chacha20() + print("\n""table") + table.test_encryption() + +if __name__ == '__main__': + main() + diff --git a/shadowsocks/server.py b/shadowsocks/server.py index bcccdb5c..d919092a 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -54,6 +54,7 @@ def main(): for port, password in config['port_password'].items(): a_config = config.copy() ipv6_ok = False + logging.info("server start with password [%s] method [%s]" % (password, a_config['method'])) if 'server_ipv6' in a_config: try: if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": @@ -70,16 +71,16 @@ def main(): except Exception as e: shell.print_exception(e) - if not ipv6_ok: - try: - a_config = config.copy() - a_config['server_port'] = int(port) - a_config['password'] = password - logging.info("starting server at %s:%d" % - (a_config['server'], int(port))) - tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) - udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) - except Exception as e: + try: + a_config = config.copy() + a_config['server_port'] = int(port) + a_config['password'] = password + logging.info("starting server at %s:%d" % + (a_config['server'], int(port))) + tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) + udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) + except Exception as e: + if not ipv6_ok: shell.print_exception(e) def run_server(): From 3af426b13b2caf7a1f51c967175fe897876a1c8b Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 30 Jun 2015 15:34:02 +0800 Subject: [PATCH 017/487] remove tail --- run.sh | 1 - shadowsocks/run.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/run.sh b/run.sh index a4698529..c8bc4516 100644 --- a/run.sh +++ b/run.sh @@ -3,4 +3,3 @@ eval $(ps -ef | grep "[0-9] python server\\.py" | awk '{print "kill "$2}') nohup python server.py >> ssserver.log 2>&1 & -tail -f ssserver.log diff --git a/shadowsocks/run.sh b/shadowsocks/run.sh index d1aa82c4..8d61b8ed 100644 --- a/shadowsocks/run.sh +++ b/shadowsocks/run.sh @@ -3,4 +3,3 @@ eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') nohup python server.py a >> ssserver.log 2>&1 & -tail -f ssserver.log From a142b957a780f3c762f33e188f4a472e66db7d59 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 1 Jul 2015 10:50:28 +0800 Subject: [PATCH 018/487] remove 'plan' & 'switch' --- db_transfer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 3e95f5c3..680ed155 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -76,7 +76,7 @@ def push_db_all_user(self): @staticmethod def pull_db_all_user(): #数据库所有用户信息 - keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'switch', 'enable', 'plan' ] + keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') cur = conn.cursor() @@ -96,10 +96,13 @@ def del_server_out_of_bound_safe(last_rows, rows): #停止超流量的服务 #启动没超流量的服务 #需要动态载入switchrule，以便实时修改规则 + try: + import switchrule + except Exception, e: + logging.error('load switchrule.py fail') cur_servers = {} for row in rows: try: - import switchrule allow = switchrule.isTurnOn(row) and row['enable'] == 1 and row['u'] + row['d'] < row['transfer_enable'] except Exception, e: allow = False From 298e8a06c9eb7a70f1249ec0265d44289db781cc Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 1 Jul 2015 10:55:24 +0800 Subject: [PATCH 019/487] add getKeys in switchrule.py --- db_transfer.py | 6 +++++- switchrule.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index 680ed155..65665273 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -76,7 +76,11 @@ def push_db_all_user(self): @staticmethod def pull_db_all_user(): #数据库所有用户信息 - keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] + try: + import switchrule + keys = switchrule.getKeys() + except Exception, e: + keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') cur = conn.cursor() diff --git a/switchrule.py b/switchrule.py index 2a0b324f..036c61bb 100644 --- a/switchrule.py +++ b/switchrule.py @@ -1,3 +1,6 @@ +def getKeys(): + return ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] + def isTurnOn(row): return True From f78f5b56e3cd3f8e8ab0f15ce449a66731e55c20 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Thu, 2 Jul 2015 15:21:05 +0800 Subject: [PATCH 020/487] client_key change --- shadowsocks/udprelay.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 09f077fd..238bf59d 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -159,9 +159,11 @@ def _handle_server(self): if not data: logging.debug('UDP handle_server: data is empty after decrypt') return - data = pre_parse_header(data) - if data is None: - return + + if not self._is_local: + data = pre_parse_header(data) + if data is None: + return header_result = parse_header(data) if header_result is None: @@ -170,10 +172,16 @@ def _handle_server(self): if self._is_local: server_addr, server_port = self._get_a_server() + key = client_key(r_addr[0], r_addr[1], dest_addr, dest_port) else: server_addr, server_port = dest_addr, dest_port + addrs = socket.getaddrinfo(dest_addr, dest_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) + if addrs: + af, socktype, proto, canonname, sa = addrs[0] + key = client_key(r_addr[0], r_addr[1], af, 0) + else: + key = None - key = client_key(r_addr[0], r_addr[1], dest_addr, dest_port) client = self._cache.get(key, None) if not client: # TODO async getaddrinfo From 69d369268ad54c937d7153967428b6a726340da1 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Thu, 9 Jul 2015 16:42:46 +0800 Subject: [PATCH 021/487] update run.sh, set working dir --- run.sh | 2 +- shadowsocks/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index c8bc4516..2925a5ec 100644 --- a/run.sh +++ b/run.sh @@ -1,5 +1,5 @@ #!/bin/bash - +cd `dirname $0` eval $(ps -ef | grep "[0-9] python server\\.py" | awk '{print "kill "$2}') nohup python server.py >> ssserver.log 2>&1 & diff --git a/shadowsocks/run.sh b/shadowsocks/run.sh index 8d61b8ed..497ceb84 100644 --- a/shadowsocks/run.sh +++ b/shadowsocks/run.sh @@ -1,5 +1,5 @@ #!/bin/bash - +cd `dirname $0` eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') nohup python server.py a >> ssserver.log 2>&1 & From 469d9f7bfaa93a07caedd5cd3f981dec34fff3ba Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 14 Jul 2015 19:02:52 +0800 Subject: [PATCH 022/487] merge master --- shadowsocks/udprelay.py | 53 +++++++++++++++++---------------- tests/jenkins.sh | 2 +- utils/fail2ban/shadowsocks.conf | 5 ++++ 3 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 utils/fail2ban/shadowsocks.conf diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 238bf59d..1e142bfa 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -76,8 +76,9 @@ BUF_SIZE = 65536 -def client_key(a, b, c, d): - return '%s:%s:%s:%s' % (a, b, c, d) +def client_key(source_addr, server_af): + # notice this is server af, not dest af + return '%s:%s:%d' % (source_addr[0], source_addr[1], server_af) class UDPRelay(object): @@ -102,6 +103,7 @@ def __init__(self, config, dns_resolver, is_local): close_callback=self._close_client) self._client_fd_to_server_addr = \ lru_cache.LRUCache(timeout=config['timeout']) + self._dns_cache = lru_cache.LRUCache(timeout=300) self._eventloop = None self._closed = False self._last_time = time.time() @@ -172,37 +174,36 @@ def _handle_server(self): if self._is_local: server_addr, server_port = self._get_a_server() - key = client_key(r_addr[0], r_addr[1], dest_addr, dest_port) else: server_addr, server_port = dest_addr, dest_port - addrs = socket.getaddrinfo(dest_addr, dest_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) - if addrs: - af, socktype, proto, canonname, sa = addrs[0] - key = client_key(r_addr[0], r_addr[1], af, 0) - else: - key = None - client = self._cache.get(key, None) - if not client: - # TODO async getaddrinfo - #logging.info('UDP handle_server %s:%d from %s:%d' % (common.to_str(server_addr), server_port, self._listen_addr, self._listen_port)) + addrs = self._dns_cache.get(server_addr, None) + if addrs is None: addrs = socket.getaddrinfo(server_addr, server_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) - if addrs: - af, socktype, proto, canonname, sa = addrs[0] - if self._forbidden_iplist: - if common.to_str(sa[0]) in self._forbidden_iplist: - logging.debug('IP %s is in forbidden list, drop' % - common.to_str(sa[0])) - # drop - return - client = socket.socket(af, socktype, proto) - client.setblocking(False) - self._cache[key] = client - self._client_fd_to_server_addr[client.fileno()] = r_addr - else: + if not addrs: # drop return + else: + self._dns_cache[server_addr] = addrs + + af, socktype, proto, canonname, sa = addrs[0] + key = client_key(r_addr, af) + logging.debug(key) + client = self._cache.get(key, None) + if not client: + # TODO async getaddrinfo + if self._forbidden_iplist: + if common.to_str(sa[0]) in self._forbidden_iplist: + logging.debug('IP %s is in forbidden list, drop' % + common.to_str(sa[0])) + # drop + return + client = socket.socket(af, socktype, proto) + client.setblocking(False) + self._cache[key] = client + self._client_fd_to_server_addr[client.fileno()] = r_addr + self._sockets.add(client.fileno()) self._eventloop.add(client, eventloop.POLL_IN) diff --git a/tests/jenkins.sh b/tests/jenkins.sh index 71d5b1ca..ea5c1630 100755 --- a/tests/jenkins.sh +++ b/tests/jenkins.sh @@ -69,7 +69,7 @@ if [ -f /proc/sys/net/ipv4/tcp_fastopen ] ; then fi run_test tests/test_large_file.sh - +run_test tests/test_udp_src.sh run_test tests/test_command.sh coverage combine && coverage report --include=shadowsocks/* diff --git a/utils/fail2ban/shadowsocks.conf b/utils/fail2ban/shadowsocks.conf new file mode 100644 index 00000000..9b1c7ec7 --- /dev/null +++ b/utils/fail2ban/shadowsocks.conf @@ -0,0 +1,5 @@ +[Definition] + +_daemon = shadowsocks + +failregex = ^\s+ERROR\s+can not parse header when handling connection from :\d+$ From a9ea55c396e2bce9231ad91a7840e0074f081497 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 5 Aug 2015 16:24:14 +0800 Subject: [PATCH 023/487] new UDP over TCP protocol, merge master --- shadowsocks/asyncdns.py | 60 ++- shadowsocks/common.py | 15 +- shadowsocks/eventloop.py | 120 ++--- shadowsocks/tcprelay.py | 209 ++++---- shadowsocks/udprelay.py | 1010 +++++++++++++++++++++++++++++++++++++- 5 files changed, 1199 insertions(+), 215 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 7e4a4ed7..c5fc99d8 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -18,7 +18,6 @@ from __future__ import absolute_import, division, print_function, \ with_statement -import time import os import socket import struct @@ -256,7 +255,6 @@ def __init__(self): self._hostname_to_cb = {} self._cb_to_hostname = {} self._cache = lru_cache.LRUCache(timeout=300) - self._last_time = time.time() self._sock = None self._servers = None self._parse_resolv() @@ -304,7 +302,7 @@ def _parse_hosts(self): except IOError: self._hosts['localhost'] = '127.0.0.1' - def add_to_loop(self, loop, ref=False): + def add_to_loop(self, loop): if self._loop: raise Exception('already add to loop') self._loop = loop @@ -312,8 +310,8 @@ def add_to_loop(self, loop, ref=False): self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP) self._sock.setblocking(False) - loop.add(self._sock, eventloop.POLL_IN) - loop.add_handler(self.handle_events, ref=ref) + loop.add(self._sock, eventloop.POLL_IN, self) + loop.add_periodic(self.handle_periodic) def _call_callback(self, hostname, ip, error=None): callbacks = self._hostname_to_cb.get(hostname, []) @@ -354,30 +352,27 @@ def _handle_data(self, data): self._call_callback(hostname, None) break - def handle_events(self, events): - for sock, fd, event in events: - if sock != self._sock: - continue - if event & eventloop.POLL_ERR: - logging.error('dns socket err') - self._loop.remove(self._sock) - self._sock.close() - # TODO when dns server is IPv6 - self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, - socket.SOL_UDP) - self._sock.setblocking(False) - self._loop.add(self._sock, eventloop.POLL_IN) - else: - data, addr = sock.recvfrom(1024) - if addr[0] not in self._servers: - logging.warn('received a packet other than our dns') - break - self._handle_data(data) - break - now = time.time() - if now - self._last_time > CACHE_SWEEP_INTERVAL: - self._cache.sweep() - self._last_time = now + def handle_event(self, sock, fd, event): + if sock != self._sock: + return + if event & eventloop.POLL_ERR: + logging.error('dns socket err') + self._loop.remove(self._sock) + self._sock.close() + # TODO when dns server is IPv6 + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + self._sock.setblocking(False) + self._loop.add(self._sock, eventloop.POLL_IN, self) + else: + data, addr = sock.recvfrom(1024) + if addr[0] not in self._servers: + logging.warn('received a packet other than our dns') + return + self._handle_data(data) + + def handle_periodic(self): + self._cache.sweep() def remove_callback(self, callback): hostname = self._cb_to_hostname.get(callback) @@ -430,6 +425,9 @@ def resolve(self, hostname, callback): def close(self): if self._sock: + if self._loop: + self._loop.remove_periodic(self.handle_periodic) + self._loop.remove(self._sock) self._sock.close() self._sock = None @@ -437,7 +435,7 @@ def close(self): def test(): dns_resolver = DNSResolver() loop = eventloop.EventLoop() - dns_resolver.add_to_loop(loop, ref=True) + dns_resolver.add_to_loop(loop) global counter counter = 0 @@ -451,8 +449,8 @@ def callback(result, error): print(result, error) counter += 1 if counter == 9: - loop.remove_handler(dns_resolver.handle_events) dns_resolver.close() + loop.stop() a_callback = callback return a_callback diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 605fbfac..11b06225 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -151,6 +151,15 @@ def pre_parse_header(data): data = data[rand_data_size + 2:] elif datatype == 0x81: data = data[1:] + elif datatype == 0x82 : + if len(data) <= 3: + return None + rand_data_size = struct.unpack('>H', data[1:3])[0] + if rand_data_size + 3 >= len(data): + logging.warn('header too short, maybe wrong password or ' + 'encryption method') + return None + data = data[rand_data_size + 3:] return data def parse_header(data): @@ -158,8 +167,8 @@ def parse_header(data): dest_addr = None dest_port = None header_length = 0 - connecttype = (addrtype & 8) and 1 or 0 - addrtype &= ~8 + connecttype = (addrtype & 0x10) and 1 or 0 + addrtype &= ~0x10 if addrtype == ADDRTYPE_IPV4: if len(data) >= 7: dest_addr = socket.inet_ntoa(data[1:5]) @@ -173,7 +182,7 @@ def parse_header(data): if len(data) >= 2 + addrlen: dest_addr = data[2:2 + addrlen] dest_port = struct.unpack('>H', data[2 + addrlen:4 + - addrlen])[0] + addrlen])[0] header_length = 4 + addrlen else: logging.warn('header is too short') diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 42f9205b..b27afe30 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -22,6 +22,7 @@ with_statement import os +import time import socket import select import errno @@ -51,23 +52,8 @@ POLL_NVAL: 'POLL_NVAL', } - -class EpollLoop(object): - - def __init__(self): - self._epoll = select.epoll() - - def poll(self, timeout): - return self._epoll.poll(timeout) - - def add_fd(self, fd, mode): - self._epoll.register(fd, mode) - - def remove_fd(self, fd): - self._epoll.unregister(fd) - - def modify_fd(self, fd, mode): - self._epoll.modify(fd, mode) +# we check timeouts every TIMEOUT_PRECISION seconds +TIMEOUT_PRECISION = 10 class KqueueLoop(object): @@ -100,17 +86,17 @@ def poll(self, timeout): results[fd] |= POLL_OUT return results.items() - def add_fd(self, fd, mode): + def register(self, fd, mode): self._fds[fd] = mode self._control(fd, mode, select.KQ_EV_ADD) - def remove_fd(self, fd): + def unregister(self, fd): self._control(fd, self._fds[fd], select.KQ_EV_DELETE) del self._fds[fd] - def modify_fd(self, fd, mode): - self.remove_fd(fd) - self.add_fd(fd, mode) + def modify(self, fd, mode): + self.unregister(fd) + self.register(fd, mode) class SelectLoop(object): @@ -129,7 +115,7 @@ def poll(self, timeout): results[fd] |= p[1] return results.items() - def add_fd(self, fd, mode): + def register(self, fd, mode): if mode & POLL_IN: self._r_list.add(fd) if mode & POLL_OUT: @@ -137,7 +123,7 @@ def add_fd(self, fd, mode): if mode & POLL_ERR: self._x_list.add(fd) - def remove_fd(self, fd): + def unregister(self, fd): if fd in self._r_list: self._r_list.remove(fd) if fd in self._w_list: @@ -145,16 +131,15 @@ def remove_fd(self, fd): if fd in self._x_list: self._x_list.remove(fd) - def modify_fd(self, fd, mode): - self.remove_fd(fd) - self.add_fd(fd, mode) + def modify(self, fd, mode): + self.unregister(fd) + self.register(fd, mode) class EventLoop(object): def __init__(self): - self._iterating = False if hasattr(select, 'epoll'): - self._impl = EpollLoop() + self._impl = select.epoll() model = 'epoll' elif hasattr(select, 'kqueue'): self._impl = KqueueLoop() @@ -165,72 +150,71 @@ def __init__(self): else: raise Exception('can not find any available functions in select ' 'package') - self._fd_to_f = {} - self._handlers = [] - self._ref_handlers = [] - self._handlers_to_remove = [] + self._fdmap = {} # (f, handler) + self._last_time = time.time() + self._periodic_callbacks = [] + self._stopping = False logging.debug('using event model: %s', model) def poll(self, timeout=None): events = self._impl.poll(timeout) - return [(self._fd_to_f[fd], fd, event) for fd, event in events] + return [(self._fdmap[fd][0], fd, event) for fd, event in events] - def add(self, f, mode): + def add(self, f, mode, handler): fd = f.fileno() - self._fd_to_f[fd] = f - self._impl.add_fd(fd, mode) + self._fdmap[fd] = (f, handler) + self._impl.register(fd, mode) def remove(self, f): fd = f.fileno() - del self._fd_to_f[fd] - self._impl.remove_fd(fd) + del self._fdmap[fd] + self._impl.unregister(fd) + + def add_periodic(self, callback): + self._periodic_callbacks.append(callback) + + def remove_periodic(self, callback): + self._periodic_callbacks.remove(callback) def modify(self, f, mode): fd = f.fileno() - self._impl.modify_fd(fd, mode) - - def add_handler(self, handler, ref=True): - self._handlers.append(handler) - if ref: - # when all ref handlers are removed, loop stops - self._ref_handlers.append(handler) - - def remove_handler(self, handler): - if handler in self._ref_handlers: - self._ref_handlers.remove(handler) - if self._iterating: - self._handlers_to_remove.append(handler) - else: - self._handlers.remove(handler) + self._impl.modify(fd, mode) + + def stop(self): + self._stopping = True def run(self): events = [] - while self._ref_handlers: + while not self._stopping: + asap = False try: - events = self.poll(1) + events = self.poll(TIMEOUT_PRECISION) except (OSError, IOError) as e: if errno_from_exception(e) in (errno.EPIPE, errno.EINTR): # EPIPE: Happens when the client closes the connection # EINTR: Happens when received a signal # handles them as soon as possible + asap = True logging.debug('poll:%s', e) else: logging.error('poll:%s', e) import traceback traceback.print_exc() continue - self._iterating = True - for handler in self._handlers: - # TODO when there are a lot of handlers - try: - handler(events) - except (OSError, IOError) as e: - shell.print_exception(e) - if self._handlers_to_remove: - for handler in self._handlers_to_remove: - self._handlers.remove(handler) - self._handlers_to_remove = [] - self._iterating = False + + for sock, fd, event in events: + handler = self._fdmap.get(fd, None) + if handler is not None: + handler = handler[1] + try: + handler.handle_event(sock, fd, event) + except (OSError, IOError) as e: + shell.print_exception(e) + now = time.time() + if asap or now - self._last_time >= TIMEOUT_PRECISION: + for callback in self._periodic_callbacks: + callback() + self._last_time = now # from tornado diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 6bb6bbc9..8188a005 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -115,6 +115,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] + self._udp_data_send_buffer = '' self._upstream_status = WAIT_STATUS_READING self._downstream_status = WAIT_STATUS_INIT self._client_address = local_sock.getpeername()[:2] @@ -128,7 +129,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) + loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR, + self._server) self.last_activity = 0 self._update_activity() @@ -185,6 +187,8 @@ def _update_stream(self, stream, status): if self._upstream_status & WAIT_STATUS_WRITING: event |= eventloop.POLL_OUT self._loop.modify(self._remote_sock, event) + if self._remote_sock_v6: + self._loop.modify(self._remote_sock_v6, event) def _write_to_sock(self, data, sock): # write data to sock @@ -193,51 +197,70 @@ def _write_to_sock(self, data, sock): if not data or not sock: return False #logging.debug("_write_to_sock %s %s %s" % (self._remote_sock, sock, self._remote_udp)) - if self._remote_udp and self._remote_sock == sock: + uncomplete = False + if self._remote_udp and sock == self._remote_sock: try: - frag = common.ord(data[2]) - if frag != 0: - logging.warn('drop a message since frag is %d' % (frag,)) - return False - else: - data = data[3:] - header_result = parse_header(data) - if header_result is None: - return False - connecttype, dest_addr, dest_port, header_length = header_result - addrs = socket.getaddrinfo(dest_addr, dest_port, 0, - socket.SOCK_DGRAM, socket.SOL_UDP) - if addrs: - af, socktype, proto, canonname, server_addr = addrs[0] - data = data[header_length:] - if af == socket.AF_INET6: - self._remote_sock_v6.sendto(data, (server_addr[0], dest_port)) + self._udp_data_send_buffer += data + #logging.info('UDP over TCP sendto %d %s' % (len(data), binascii.hexlify(data))) + while len(self._udp_data_send_buffer) > 6: + length = struct.unpack('>H', self._udp_data_send_buffer[:2])[0] + + if length > len(self._udp_data_send_buffer): + break + + data = self._udp_data_send_buffer[:length] + self._udp_data_send_buffer = self._udp_data_send_buffer[length:] + + frag = common.ord(data[2]) + if frag != 0: + logging.warn('drop a message since frag is %d' % (frag,)) + continue else: - sock.sendto(data, (server_addr[0], dest_port)) + data = data[3:] + header_result = parse_header(data) + if header_result is None: + continue + connecttype, dest_addr, dest_port, header_length = header_result + addrs = socket.getaddrinfo(dest_addr, dest_port, 0, + socket.SOCK_DGRAM, socket.SOL_UDP) + #logging.info('UDP over TCP sendto %s:%d %d bytes from %s:%d' % (dest_addr, dest_port, len(data), self._client_address[0], self._client_address[1])) + if addrs: + af, socktype, proto, canonname, server_addr = addrs[0] + data = data[header_length:] + if af == socket.AF_INET6: + self._remote_sock_v6.sendto(data, (server_addr[0], dest_port)) + else: + sock.sendto(data, (server_addr[0], dest_port)) except Exception as e: #trace = traceback.format_exc() #logging.error(trace) - logging.error(e) + error_no = eventloop.errno_from_exception(e) + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): + uncomplete = True + else: + shell.print_exception(e) + self.destroy() + return False return True - - uncomplete = False - try: - l = len(data) - s = sock.send(data) - if s < l: - data = data[s:] - uncomplete = True - except (OSError, IOError) as e: - error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS, - errno.EWOULDBLOCK): - uncomplete = True - else: - #traceback.print_exc() - shell.print_exception(e) - self.destroy() - return False + else: + try: + l = len(data) + s = sock.send(data) + if s < l: + data = data[s:] + uncomplete = True + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): + uncomplete = True + else: + #traceback.print_exc() + shell.print_exception(e) + self.destroy() + return False if uncomplete: if sock == self._local_sock: self._data_to_write_to_local.append(data) @@ -270,7 +293,7 @@ def _handle_stage_connecting(self, data): remote_sock = \ self._create_remote_socket(self._chosen_server[0], self._chosen_server[1]) - self._loop.add(remote_sock, eventloop.POLL_ERR) + self._loop.add(remote_sock, eventloop.POLL_ERR, self._server) data = b''.join(self._data_to_write_to_remote) l = len(data) s = remote_sock.sendto(data, MSG_FASTOPEN, self._chosen_server) @@ -382,6 +405,11 @@ def _create_remote_socket(self, ip, port): remote_sock_v6 = socket.socket(af, socktype, proto) self._remote_sock_v6 = remote_sock_v6 self._fd_to_handlers[remote_sock_v6.fileno()] = self + remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) + remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) + remote_sock_v6.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) + remote_sock_v6.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) + remote_sock.setblocking(False) if self._remote_udp: @@ -421,10 +449,12 @@ def _handle_dns_resolved(self, result, error): remote_port) if self._remote_udp: self._loop.add(remote_sock, - eventloop.POLL_IN) + eventloop.POLL_IN, + self._server) if self._remote_sock_v6: self._loop.add(self._remote_sock_v6, - eventloop.POLL_IN) + eventloop.POLL_IN, + self._server) else: try: remote_sock.connect((remote_addr, remote_port)) @@ -433,10 +463,16 @@ def _handle_dns_resolved(self, result, error): errno.EINPROGRESS: pass self._loop.add(remote_sock, - eventloop.POLL_ERR | eventloop.POLL_OUT) + eventloop.POLL_ERR | eventloop.POLL_OUT, + self._server) self._stage = STAGE_CONNECTING self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + if self._remote_udp: + while self._data_to_write_to_remote: + data = self._data_to_write_to_remote[0] + del self._data_to_write_to_remote[0] + self._write_to_sock(data, self._remote_sock) return except Exception as e: shell.print_exception(e) @@ -495,11 +531,12 @@ def _on_remote_read(self, is_remote_sock): port = struct.pack('>H', addr[1]) try: ip = socket.inet_aton(addr[0]) - data = '\x00\x00\x00\x01' + ip + port + data + data = '\x00\x01' + ip + port + data except Exception as e: ip = socket.inet_pton(socket.AF_INET6, addr[0]) - data = '\x00\x00\x00\x04' + ip + port + data - logging.info('UDP recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) + data = '\x00\x04' + ip + port + data + data = struct.pack('>H', len(data) + 2) + data + #logging.info('UDP over TCP recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) else: data = self._remote_sock.recv(BUF_SIZE) except (OSError, IOError) as e: @@ -637,7 +674,6 @@ def __init__(self, config, dns_resolver, is_local): self._closed = False self._eventloop = None self._fd_to_handlers = {} - self._last_time = time.time() self.server_transfer_ul = 0L self.server_transfer_dl = 0L @@ -680,10 +716,9 @@ def add_to_loop(self, loop): if self._closed: raise Exception('already closed') self._eventloop = loop - loop.add_handler(self._handle_events) - self._eventloop.add(self._server_socket, - eventloop.POLL_IN | eventloop.POLL_ERR) + eventloop.POLL_IN | eventloop.POLL_ERR, self) + self._eventloop.add_periodic(self.handle_periodic) def remove_handler(self, handler): index = self._handler_to_timeouts.get(hash(handler), -1) @@ -695,7 +730,7 @@ def remove_handler(self, handler): def update_activity(self, handler): # set handler to active now = int(time.time()) - if now - handler.last_activity < TIMEOUT_PRECISION: + if now - handler.last_activity < eventloop.TIMEOUT_PRECISION: # thus we can lower timeout modification frequency return handler.last_activity = now @@ -741,53 +776,55 @@ def _sweep_timeout(self): pos = 0 self._timeout_offset = pos - def _handle_events(self, events): + def handle_event(self, sock, fd, event): # handle events and dispatch to handlers - for sock, fd, event in events: + if sock: + logging.log(shell.VERBOSE_LEVEL, 'fd %d %s', fd, + eventloop.EVENT_NAMES.get(event, event)) + if sock == self._server_socket: + if event & eventloop.POLL_ERR: + # TODO + raise Exception('server_socket error') + try: + logging.debug('accept') + conn = self._server_socket.accept() + TCPRelayHandler(self, self._fd_to_handlers, + self._eventloop, conn[0], self._config, + self._dns_resolver, self._is_local) + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): + return + else: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() + else: if sock: - logging.log(shell.VERBOSE_LEVEL, 'fd %d %s', fd, - eventloop.EVENT_NAMES.get(event, event)) - if sock == self._server_socket: - if event & eventloop.POLL_ERR: - # TODO - raise Exception('server_socket error') - try: - logging.debug('accept') - conn = self._server_socket.accept() - TCPRelayHandler(self, self._fd_to_handlers, - self._eventloop, conn[0], self._config, - self._dns_resolver, self._is_local) - except (OSError, IOError) as e: - error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS, - errno.EWOULDBLOCK): - continue - else: - shell.print_exception(e) - if self._config['verbose']: - traceback.print_exc() + handler = self._fd_to_handlers.get(fd, None) + if handler: + handler.handle_event(sock, event) else: - if sock: - handler = self._fd_to_handlers.get(fd, None) - if handler: - handler.handle_event(sock, event) - else: - logging.warn('poll removed fd') + logging.warn('poll removed fd') - now = time.time() - if now - self._last_time > TIMEOUT_PRECISION: - self._sweep_timeout() - self._last_time = now + def handle_periodic(self): if self._closed: if self._server_socket: self._eventloop.remove(self._server_socket) self._server_socket.close() self._server_socket = None - logging.info('closed listen port %d', self._listen_port) + logging.info('closed TCP port %d', self._listen_port) if not self._fd_to_handlers: - self._eventloop.remove_handler(self._handle_events) + logging.info('stopping') + self._eventloop.stop() + self._sweep_timeout() def close(self, next_tick=False): + logging.debug('TCP close') self._closed = True if not next_tick: + if self._eventloop: + self._eventloop.remove_periodic(self.handle_periodic) + self._eventloop.remove(self._server_socket) self._server_socket.close() diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 1e142bfa..018a6a6d 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -68,19 +68,760 @@ import struct import errno import random +import binascii +import traceback from shadowsocks import encrypt, eventloop, lru_cache, common, shell from shadowsocks.common import pre_parse_header, parse_header, pack_addr +# we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time +TIMEOUTS_CLEAN_SIZE = 512 + +# we check timeouts every TIMEOUT_PRECISION seconds +TIMEOUT_PRECISION = 4 + +# for each handler, we have 2 stream directions: +# upstream: from client to server direction +# read local and write to remote +# downstream: from server to client direction +# read remote and write to local + +STREAM_UP = 0 +STREAM_DOWN = 1 + +# for each stream, it's waiting for reading, or writing, or both +WAIT_STATUS_INIT = 0 +WAIT_STATUS_READING = 1 +WAIT_STATUS_WRITING = 2 +WAIT_STATUS_READWRITING = WAIT_STATUS_READING | WAIT_STATUS_WRITING BUF_SIZE = 65536 +DOUBLE_SEND_BEG_IDS = 16 +POST_MTU_MIN = 1000 +POST_MTU_MAX = 1400 + +STAGE_INIT = 0 +STAGE_RSP_ID = 1 +STAGE_DNS = 2 +STAGE_CONNECTING = 3 +STAGE_STREAM = 4 +STAGE_DESTROYED = -1 + +CMD_CONNECT = 0 +CMD_RSP_CONNECT = 1 +CMD_CONNECT_REMOTE = 2 +CMD_RSP_CONNECT_REMOTE = 3 +CMD_POST = 4 +CMD_SYN_STATUS = 5 +CMD_POST_64 = 6 +CMD_SYN_STATUS_64 = 7 +CMD_DISCONNECT = 8 + +CMD_VER_STR = "\x08" + +class UDPLocalAddress(object): + def __init__(self, addr): + self.addr = addr + self.last_activity = time.time() + + def is_timeout(self): + return time.time() - self.last_activity > 30 + +class PacketInfo(object): + def __init__(self, data): + self.data = data + self.time = time.time() + +class SendingQueue(object): + def __init__(self): + self.queue = {} + self.begin_id = 0 + self.end_id = 1 + self.interval = 0.5 + + def append(self, data): + self.queue[self.end_id] = PacketInfo(data) + self.end_id += 1 + return self.end_id - 1 + + def empty(self): + return self.begin_id + 1 == self.end_id + + def size(self): + return self.end_id - self.begin_id - 1 + + def get_begin_id(self): + return self.begin_id + + def get_end_id(self): + return self.end_id + + def get_data_list(self, pack_id_base, pack_id_list): + ret_list = [] + curtime = time.time() + for pack_id in pack_id_list: + offset = pack_id_base + pack_id + if offset <= self.begin_id or self.end_id <= offset: + continue + ret_data = self.queue[offset] + if curtime - ret_data.time > self.interval: + ret_data.time = curtime + ret_list.append( (offset, ret_data.data) ) + return ret_list + + def set_finish(self, begin_id, done_list): + while self.begin_id < begin_id: + self.begin_id += 1 + del self.queue[self.begin_id] + #while len(self.queue) > 0 and self.queue[0][0] <= begin_id: + # del self.queue[0] + # self.begin_id += 1 + +class RecvQueue(object): + def __init__(self): + self.queue = {} + self.miss_queue = set() + self.begin_id = 0 + self.end_id = 1 + + def empty(self): + return self.begin_id + 1 == self.end_id + + def insert(self, pack_id, data): + if (pack_id not in self.queue) and pack_id > self.begin_id: + self.queue[pack_id] = PacketInfo(data) + if self.end_id == pack_id: + self.end_id = pack_id + 1 + elif self.end_id < pack_id: + for eid in xrange(self.end_id, pack_id): + self.miss_queue.add(eid) + self.end_id = pack_id + 1 + else: + self.miss_queue.remove(pack_id) + + def set_end(self, end_id): + if end_id > self.end_id: + for eid in xrange(self.end_id, end_id): + self.miss_queue.add(eid) + self.end_id = end_id + + def get_begin_id(self): + return self.begin_id + + def has_data(self): + return (self.begin_id + 1) in self.queue + + def get_data(self): + if (self.begin_id + 1) in self.queue: + self.begin_id += 1 + pack_id = self.begin_id + ret_data = self.queue[pack_id] + del self.queue[pack_id] + return (pack_id, ret_data.data) + + def get_missing_id(self, begin_id): + missing = [] + if begin_id == 0: + begin_id = self.begin_id + for i in self.miss_queue: + if i - begin_id > 32768: + break + missing.append(i - begin_id) + return (begin_id, missing) + +class TCPRelayHandler(object): + def __init__(self, server, reqid_to_handlers, fd_to_handlers, loop, + local_sock, local_id, client_param, config, + dns_resolver, is_local): + self._server = server + self._reqid_to_handlers = reqid_to_handlers + self._fd_to_handlers = fd_to_handlers + self._loop = loop + self._local_sock = local_sock + self._remote_sock = None + self._remote_udp = False + self._config = config + self._dns_resolver = dns_resolver + self._local_id = local_id + + self._is_local = is_local + self._stage = STAGE_INIT + self._password = config['password'] + self._method = config['method'] + self._fastopen_connected = False + self._data_to_write_to_local = [] + self._data_to_write_to_remote = [] + self._upstream_status = WAIT_STATUS_READING + self._downstream_status = WAIT_STATUS_INIT + self._request_id = 0 + self._client_address = {} + self._remote_address = None + self._sendingqueue = SendingQueue() + self._recvqueue = RecvQueue() + if 'forbidden_ip' in config: + self._forbidden_iplist = config['forbidden_ip'] + else: + self._forbidden_iplist = None + #fd_to_handlers[local_sock.fileno()] = self + #local_sock.setblocking(False) + #loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) + self.last_activity = 0 + self._update_activity() + self._random_mtu_size = [random.randint(POST_MTU_MIN, POST_MTU_MAX) for i in xrange(1024)] + self._random_mtu_index = 0 + + self._rand_data = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" * 4 + + def __hash__(self): + # default __hash__ is id / 16 + # we want to eliminate collisions + return id(self) + + @property + def remote_address(self): + return self._remote_address + + def add_local_address(self, addr): + self._client_address[addr] = UDPLocalAddress(addr) + + def _update_activity(self): + # tell the TCP Relay we have activities recently + # else it will think we are inactive and timed out + self._server.update_activity(self) + + def _update_stream(self, stream, status): + # update a stream to a new waiting status + + # check if status is changed + # only update if dirty + dirty = False + if stream == STREAM_DOWN: + if self._downstream_status != status: + self._downstream_status = status + dirty = True + elif stream == STREAM_UP: + if self._upstream_status != status: + self._upstream_status = status + dirty = True + if dirty: + ''' + if self._local_sock: + event = eventloop.POLL_ERR + if self._downstream_status & WAIT_STATUS_WRITING: + event |= eventloop.POLL_OUT + if self._upstream_status & WAIT_STATUS_READING: + event |= eventloop.POLL_IN + self._loop.modify(self._local_sock, event) + ''' + if self._remote_sock: + event = eventloop.POLL_ERR + if self._downstream_status & WAIT_STATUS_READING: + event |= eventloop.POLL_IN + if self._upstream_status & WAIT_STATUS_WRITING: + event |= eventloop.POLL_OUT + self._loop.modify(self._remote_sock, event) + + def _write_to_sock(self, data, sock, addr = None): + # write data to sock + # if only some of the data are written, put remaining in the buffer + # and update the stream to wait for writing + if not data or not sock: + return False + + uncomplete = False + retry = 0 + if sock == self._local_sock: + data = encrypt.encrypt_all(self._password, self._method, 1, data) + if addr is None: + return False + try: + self._server.write_to_server_socket(data, addr) + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + uncomplete = True + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): + pass + else: + #traceback.print_exc() + shell.print_exception(e) + self.destroy() + return False + else: + try: + l = len(data) + s = sock.send(data) + if s < l: + data = data[s:] + uncomplete = True + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): + uncomplete = True + else: + #logging.error(traceback.extract_stack()) + #traceback.print_exc() + shell.print_exception(e) + self.destroy() + return False + if uncomplete: + if sock == self._local_sock: + #if data is not None and retry < 10: + # self._data_to_write_to_local.append([(data, addr), retry]) + self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) + elif sock == self._remote_sock: + self._data_to_write_to_remote.append(data) + self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) + else: + logging.error('write_all_to_sock:unknown socket') + else: + if sock == self._local_sock: + if self._sendingqueue.size() > 8192: + self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) + else: + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + elif sock == self._remote_sock: + if self._sendingqueue.size() > 8192: + self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) + else: + self._update_stream(STREAM_UP, WAIT_STATUS_READING) + else: + logging.error('write_all_to_sock:unknown socket') + return True + + def _create_remote_socket(self, ip, port): + addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) + if len(addrs) == 0: + raise Exception("getaddrinfo failed for %s:%d" % (ip, port)) + af, socktype, proto, canonname, sa = addrs[0] + if self._forbidden_iplist: + if common.to_str(sa[0]) in self._forbidden_iplist: + raise Exception('IP %s is in forbidden list, reject' % + common.to_str(sa[0])) + remote_sock = socket.socket(af, socktype, proto) + self._remote_sock = remote_sock + + self._fd_to_handlers[remote_sock.fileno()] = self + + remote_sock.setblocking(False) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + return remote_sock + + def _handle_dns_resolved(self, result, error): + if error: + self._log_error(error) + self.destroy() + return + if result: + ip = result[1] + if ip: + + try: + self._stage = STAGE_CONNECTING + remote_addr = ip + remote_port = self._remote_address[1] + logging.info("connect to %s : %d" % (remote_addr, remote_port)) + + remote_sock = self._create_remote_socket(remote_addr, + remote_port) + try: + remote_sock.connect((remote_addr, remote_port)) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == \ + errno.EINPROGRESS: + pass + + self._loop.add(remote_sock, + eventloop.POLL_ERR | eventloop.POLL_OUT, + self._server) + self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + self._stage = STAGE_STREAM + + for it_addr in self._client_address: + addr = it_addr + break + + for i in xrange(2): + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, "\x02") + self._write_to_sock(rsp_data, self._local_sock, addr) + + return + except Exception as e: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() + self.destroy() + + def _on_local_read(self): + # handle all local read events and dispatch them to methods for + # each stage + self._update_activity() + if not self._local_sock: + return + data = None + try: + data = self._local_sock.recv(BUF_SIZE) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) in \ + (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): + return + if not data: + self.destroy() + return + if not data: + return + self._server.server_transfer_ul += len(data) + #TODO ============================================================ + if self._stage == STAGE_STREAM: + self._write_to_sock(data, self._remote_sock) + return + def _on_remote_read(self): + # handle all remote read events + self._update_activity() + data = None + try: + data = self._remote_sock.recv(BUF_SIZE) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) in \ + (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK, 10035): #errno.WSAEWOULDBLOCK + return + if not data: + self.destroy() + return + self._server.server_transfer_dl += len(data) + try: + recv_data = data + beg_pos = 0 + max_len = len(recv_data) + while beg_pos < max_len: + if beg_pos + POST_MTU_MAX >= max_len: + split_pos = max_len + else: + split_pos = beg_pos + self._random_mtu_size[self._random_mtu_index] + self._random_mtu_index = (self._random_mtu_index + 1) & 0x3ff + #split_pos = beg_pos + random.randint(POST_MTU_MIN, POST_MTU_MAX) + data = recv_data[beg_pos:split_pos] + beg_pos = split_pos + + pack_id = self._sendingqueue.append(data) + post_data = self._pack_post_data(CMD_POST, pack_id, data) + for it_addr in self._client_address: + addr = it_addr + break + self._write_to_sock(post_data, self._local_sock, addr) + #if pack_id <= DOUBLE_SEND_BEG_IDS: + # post_data = self._pack_post_data(CMD_POST, pack_id, data) + # self._write_to_sock(post_data, self._local_sock, addr) + + except Exception as e: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() + # TODO use logging when debug completed + self.destroy() + + def _on_local_write(self): + # handle local writable event + if self._data_to_write_to_local: + data = b''.join(self._data_to_write_to_local) + self._data_to_write_to_local = [] + self._write_to_sock(data, self._local_sock) + else: + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + + def _on_remote_write(self): + # handle remote writable event + self._stage = STAGE_STREAM + if self._data_to_write_to_remote: + data = b''.join(self._data_to_write_to_remote) + self._data_to_write_to_remote = [] + self._write_to_sock(data, self._remote_sock) + else: + self._update_stream(STREAM_UP, WAIT_STATUS_READING) + + def _on_local_error(self): + logging.debug('got local error') + if self._local_sock: + logging.error(eventloop.get_sock_error(self._local_sock)) + self.destroy() + + def _on_remote_error(self): + logging.debug('got remote error') + if self._remote_sock: + logging.error(eventloop.get_sock_error(self._remote_sock)) + self.destroy() + + def _pack_rsp_data(self, cmd, data): + reqid_str = struct.pack(">H", self._request_id) + return ''.join([CMD_VER_STR, chr(cmd), reqid_str, data, self._rand_data[:random.randint(0, len(self._rand_data))], reqid_str]) + + def _pack_rnd_data(self, data): + length = random.randint(0, len(self._rand_data)) + if length == 0: + return data + elif length == 1: + return "\x81" + data + elif length < 256: + return "\x80" + chr(length) + self._rand_data[:length - 2] + data + else: + return "\x82" + struct.pack(">H", length) + self._rand_data[:length - 3] + data + + def _pack_post_data(self, cmd, pack_id, data): + reqid_str = struct.pack(">H", self._request_id) + recv_id = self._recvqueue.get_begin_id() + rsp_data = ''.join([CMD_VER_STR, chr(cmd), reqid_str, struct.pack(">I", recv_id), struct.pack(">I", pack_id), data, reqid_str]) + return rsp_data + + def _pack_post_data_64(self, cmd, send_id, pack_id, data): + reqid_str = struct.pack(">H", self._request_id) + recv_id = self._recvqueue.get_begin_id() + rsp_data = ''.join([CMD_VER_STR, chr(cmd), reqid_str, struct.pack(">Q", recv_id), struct.pack(">Q", pack_id), data, reqid_str]) + return rsp_data + + def sweep_timeout(self): + logging.info("sweep_timeout") + if self._stage == STAGE_STREAM: + pack_id, missing = self._recvqueue.get_missing_id(0) + logging.info("sweep_timeout %s %s" % (pack_id, missing)) + data = '' + for pid in missing: + data += struct.pack(">H", pid) + rsp_data = self._pack_post_data(CMD_SYN_STATUS, pack_id, data) + self._write_to_sock(rsp_data, self._local_sock, addr) + + def handle_stream_sync_status(self, addr, cmd, request_id, pack_id, max_send_id, data): + missing_list = [] + while len(data) >= 2: + pid = struct.unpack(">H", data[0:2])[0] + data = data[2:] + missing_list.append(pid) + done_list = [] + self._recvqueue.set_end(max_send_id) + self._sendingqueue.set_finish(pack_id, done_list) + + if self._stage == STAGE_DESTROYED and self._sendingqueue.empty(): + self.destroy_local() + return + + # post CMD_SYN_STATUS + send_id = self._sendingqueue.get_end_id() + post_pack_id, missing = self._recvqueue.get_missing_id(0) + pack_ids_data = '' + for pid in missing: + pack_ids_data += struct.pack(">H", pid) + + rsp_data = self._pack_rnd_data(self._pack_post_data(CMD_SYN_STATUS, send_id, pack_ids_data)) + self._write_to_sock(rsp_data, self._local_sock, addr) + + send_list = self._sendingqueue.get_data_list(pack_id, missing_list) + for post_pack_id, post_data in send_list: + rsp_data = self._pack_post_data(CMD_POST, post_pack_id, post_data) + self._write_to_sock(rsp_data, self._local_sock, addr) + #if post_pack_id <= DOUBLE_SEND_BEG_IDS: + # rsp_data = self._pack_post_data(CMD_POST, post_pack_id, post_data) + # self._write_to_sock(rsp_data, self._local_sock, addr) + + def handle_client(self, addr, cmd, request_id, data): + self.add_local_address(addr) + if cmd == CMD_DISCONNECT: + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + self._write_to_sock(rsp_data, self._local_sock, addr) + self.destroy() + self.destroy_local() + return + if self._stage == STAGE_INIT: + if cmd == CMD_CONNECT: + self._request_id = request_id + self._stage = STAGE_RSP_ID + return + if self._request_id != request_id: + return + + if self._stage == STAGE_RSP_ID: + if cmd == CMD_CONNECT: + for i in xrange(2): + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, "\x01") + self._write_to_sock(rsp_data, self._local_sock, addr) + elif cmd == CMD_CONNECT_REMOTE: + local_id = data[0:4] + if self._local_id == local_id: + data = data[4:] + header_result = parse_header(data) + if header_result is None: + return + connecttype, remote_addr, remote_port, header_length = header_result + self._remote_address = (common.to_str(remote_addr), remote_port) + self._stage = STAGE_DNS + self._dns_resolver.resolve(remote_addr, + self._handle_dns_resolved) + logging.info('TCP connect %s:%d from %s:%d' % (remote_addr, remote_port, addr[0], addr[1])) + else: + # ileagal request + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + self._write_to_sock(rsp_data, self._local_sock, addr) + elif self._stage == STAGE_CONNECTING: + if cmd == CMD_CONNECT_REMOTE: + local_id = data[0:4] + if self._local_id == local_id: + for i in xrange(2): + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, "\x02") + self._write_to_sock(rsp_data, self._local_sock, addr) + else: + # ileagal request + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + self._write_to_sock(rsp_data, self._local_sock, addr) + elif self._stage == STAGE_STREAM: + if len(data) < 4: + # ileagal request + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + self._write_to_sock(rsp_data, self._local_sock, addr) + return + local_id = data[0:4] + if self._local_id != local_id: + # ileagal request + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + self._write_to_sock(rsp_data, self._local_sock, addr) + return + else: + data = data[4:] + if cmd == CMD_CONNECT_REMOTE: + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, "\x02") + self._write_to_sock(rsp_data, self._local_sock, addr) + elif cmd == CMD_POST: + recv_id = struct.unpack(">I", data[0:4])[0] + pack_id = struct.unpack(">I", data[4:8])[0] + self._recvqueue.insert(pack_id, data[8:]) + self._sendingqueue.set_finish(recv_id, []) + elif cmd == CMD_POST_64: + recv_id = struct.unpack(">Q", data[0:8])[0] + pack_id = struct.unpack(">Q", data[8:16])[0] + self._recvqueue.insert(pack_id, data[16:]) + self._sendingqueue.set_finish(recv_id, []) + elif cmd == CMD_DISCONNECT: + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + self._write_to_sock(rsp_data, self._local_sock, addr) + self.destroy() + self.destroy_local() + return + elif cmd == CMD_SYN_STATUS: + pack_id = struct.unpack(">I", data[0:4])[0] + max_send_id = struct.unpack(">I", data[4:8])[0] + data = data[8:] + self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) + elif cmd == CMD_SYN_STATUS_64: + pack_id = struct.unpack(">Q", data[0:8])[0] + max_send_id = struct.unpack(">Q", data[8:16])[0] + data = data[16:] + self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) + while self._recvqueue.has_data(): + pack_id, post_data = self._recvqueue.get_data() + self._write_to_sock(post_data, self._remote_sock) + elif self._stage == STAGE_DESTROYED: + local_id = data[0:4] + if self._local_id != local_id: + # ileagal request + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + self._write_to_sock(rsp_data, self._local_sock, addr) + return + else: + data = data[4:] + if cmd == CMD_SYN_STATUS: + pack_id = struct.unpack(">I", data[0:4])[0] + max_send_id = struct.unpack(">I", data[4:8])[0] + data = data[8:] + logging.info('handle_client STAGE_DESTROYED send %d %d' % (request_id, pack_id)) + self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) + elif cmd == CMD_SYN_STATUS_64: + pack_id = struct.unpack(">Q", data[0:8])[0] + max_send_id = struct.unpack(">Q", data[8:16])[0] + data = data[16:] + logging.info('handle_client STAGE_DESTROYED send %d %d' % (request_id, pack_id)) + self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) + + def handle_event(self, sock, event): + # handle all events in this handler and dispatch them to methods + if self._stage == STAGE_DESTROYED: + logging.debug('ignore handle_event: destroyed') + return + # order is important + if sock == self._remote_sock: + if event & eventloop.POLL_ERR: + self._on_remote_error() + if self._stage == STAGE_DESTROYED: + return + if event & (eventloop.POLL_IN | eventloop.POLL_HUP): + self._on_remote_read() + if self._stage == STAGE_DESTROYED: + return + if event & eventloop.POLL_OUT: + self._on_remote_write() + elif sock == self._local_sock: + if event & eventloop.POLL_ERR: + self._on_local_error() + if self._stage == STAGE_DESTROYED: + return + if event & (eventloop.POLL_IN | eventloop.POLL_HUP): + self._on_local_read() + if self._stage == STAGE_DESTROYED: + return + if event & eventloop.POLL_OUT: + self._on_local_write() + else: + logging.warn('unknown socket') + + def _log_error(self, e): + logging.error('%s when handling connection from %s' % + (e, self._client_address.keys())) + + def destroy(self): + # destroy the handler and release any resources + # promises: + # 1. destroy won't make another destroy() call inside + # 2. destroy releases resources so it prevents future call to destroy + # 3. destroy won't raise any exceptions + # if any of the promises are broken, it indicates a bug has been + # introduced! mostly likely memory leaks, etc + #logging.info('tcp destroy called') + if self._stage == STAGE_DESTROYED: + # this couldn't happen + logging.debug('already destroyed') + return + self._stage = STAGE_DESTROYED + if self._remote_address: + logging.debug('destroy: %s:%d' % + self._remote_address) + else: + logging.debug('destroy') + if self._remote_sock: + logging.debug('destroying remote') + self._loop.remove(self._remote_sock) + del self._fd_to_handlers[self._remote_sock.fileno()] + self._remote_sock.close() + self._remote_sock = None + if self._sendingqueue.empty(): + self.destroy_local() + self._dns_resolver.remove_callback(self._handle_dns_resolved) + + def destroy_local(self): + if self._local_sock: + logging.debug('disconnect local') + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + addr = None + for it_addr in self._client_address: + addr = it_addr + break + self._write_to_sock(rsp_data, self._local_sock, addr) + self._local_sock = None + del self._reqid_to_handlers[self._request_id] + self._server.remove_handler(self) def client_key(source_addr, server_af): # notice this is server af, not dest af return '%s:%s:%d' % (source_addr[0], source_addr[1], server_af) - class UDPRelay(object): def __init__(self, config, dns_resolver, is_local): self._config = config @@ -106,8 +847,19 @@ def __init__(self, config, dns_resolver, is_local): self._dns_cache = lru_cache.LRUCache(timeout=300) self._eventloop = None self._closed = False - self._last_time = time.time() + self.server_transfer_ul = 0L + self.server_transfer_dl = 0L + self._sockets = set() + self._fd_to_handlers = {} + self._reqid_to_hd = {} + self._data_to_write_to_server_socket = [] + + self._timeouts = [] # a list for all the handlers + # we trim the timeouts once a while + self._timeout_offset = 0 # last checked position for timeout + self._handler_to_timeouts = {} # key: handler value: index in timeouts + if 'forbidden_ip' in config: self._forbidden_iplist = config['forbidden_ip'] else: @@ -122,6 +874,8 @@ def __init__(self, config, dns_resolver, is_local): server_socket = socket.socket(af, socktype, proto) server_socket.bind((self._listen_addr, self._listen_port)) server_socket.setblocking(False) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) self._server_socket = server_socket def _get_a_server(self): @@ -143,6 +897,41 @@ def _close_client(self, client): # just an address pass + def _pre_parse_udp_header(self, data): + if data is None: + return + datatype = ord(data[0]) + if datatype == 0x8: + if len(data) >= 8: + crc = binascii.crc32(data) & 0xffffffff + if crc != 0xffffffff: + logging.warn('uncorrect CRC32, maybe wrong password or ' + 'encryption method') + return None + cmd = ord(data[1]) + request_id = struct.unpack('>H', data[2:4])[0] + data = data[4:-4] + return (cmd, request_id, data) + elif len(data) >= 6 and ord(data[1]) == 0x0: + crc = binascii.crc32(data) & 0xffffffff + if crc != 0xffffffff: + logging.warn('uncorrect CRC32, maybe wrong password or ' + 'encryption method') + return None + cmd = ord(data[1]) + data = data[2:-4] + return (cmd, 0, data) + else: + logging.warn('header too short, maybe wrong password or ' + 'encryption method') + return None + return data + + def _pack_rsp_data(self, cmd, request_id, data): + _rand_data = "123456789abcdefghijklmnopqrstuvwxyz" * 2 + reqid_str = struct.pack(">H", request_id) + return ''.join([CMD_VER_STR, chr(cmd), reqid_str, data, _rand_data[:random.randint(0, len(_rand_data))], reqid_str]) + def _handle_server(self): server = self._server_socket data, r_addr = server.recvfrom(BUF_SIZE) @@ -162,11 +951,75 @@ def _handle_server(self): logging.debug('UDP handle_server: data is empty after decrypt') return + #logging.info("UDP data %s" % (binascii.hexlify(data),)) if not self._is_local: data = pre_parse_header(data) + + data = self._pre_parse_udp_header(data) if data is None: return + if type(data) is tuple: + #(cmd, request_id, data) + #logging.info("UDP data %d %d %s" % (data[0], data[1], binascii.hexlify(data[2]))) + try: + if data[0] == 0: + if len(data[2]) >= 4: + for i in xrange(64): + req_id = random.randint(1, 65535) + if req_id not in self._reqid_to_hd: + break + if req_id in self._reqid_to_hd: + for i in xrange(64): + req_id = random.randint(1, 65535) + if type(self._reqid_to_hd[req_id]) is tuple: + break + # return req id + self._reqid_to_hd[req_id] = (data[2][0:4], None) + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, req_id, "\x01") + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + elif data[0] == CMD_CONNECT_REMOTE: + if len(data[2]) > 4 and data[1] in self._reqid_to_hd: + # create + if type(self._reqid_to_hd[data[1]]) is tuple: + if data[2][0:4] == self._reqid_to_hd[data[1]][0]: + handle = TCPRelayHandler(self, self._reqid_to_hd, self._fd_to_handlers, + self._eventloop, self._server_socket, + self._reqid_to_hd[data[1]][0], self._reqid_to_hd[data[1]][1], + self._config, self._dns_resolver, self._is_local) + self._reqid_to_hd[data[1]] = handle + handle.handle_client(r_addr, CMD_CONNECT, data[1], data[2]) + handle.handle_client(r_addr, *data) + self.update_activity(handle) + else: + # disconnect + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], "") + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + else: + self.update_activity(self._reqid_to_hd[data[1]]) + self._reqid_to_hd[data[1]].handle_client(r_addr, *data) + else: + # disconnect + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], "") + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + elif data[0] > CMD_CONNECT_REMOTE and data[0] <= CMD_DISCONNECT: + if data[1] in self._reqid_to_hd: + self.update_activity(self._reqid_to_hd[data[1]]) + self._reqid_to_hd[data[1]].handle_client(r_addr, *data) + else: + # disconnect + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], "") + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + return + except Exception as e: + trace = traceback.format_exc() + logging.error(trace) + return + header_result = parse_header(data) if header_result is None: return @@ -205,7 +1058,7 @@ def _handle_server(self): self._client_fd_to_server_addr[client.fileno()] = r_addr self._sockets.add(client.fileno()) - self._eventloop.add(client, eventloop.POLL_IN) + self._eventloop.add(client, eventloop.POLL_IN, self) if self._is_local: data = encrypt.encrypt_all(self._password, self._method, 1, data) @@ -216,6 +1069,7 @@ def _handle_server(self): if not data: return try: + #logging.info('UDP handle_server sendto %s:%d %d bytes' % (common.to_str(server_addr), server_port, len(data))) client.sendto(data, (server_addr, server_port)) except IOError as e: err = eventloop.errno_from_exception(e) @@ -247,50 +1101,152 @@ def _handle_client(self, sock): header_result = parse_header(data) if header_result is None: return - # connecttype, dest_addr, dest_port, header_length = header_result + connecttype, dest_addr, dest_port, header_length = header_result + #logging.debug('UDP handle_client %s:%d to %s:%d' % (common.to_str(r_addr[0]), r_addr[1], dest_addr, dest_port)) + response = b'\x00\x00\x00' + data client_addr = self._client_fd_to_server_addr.get(sock.fileno()) if client_addr: - self._server_socket.sendto(response, client_addr) + self.write_to_server_socket(response, client_addr) else: # this packet is from somewhere else we know # simply drop that packet pass + def write_to_server_socket(self, data, addr): + #self._server_socket.sendto(data, addr) + #''' + uncomplete = False + retry = 0 + try: + #""" + #if self._data_to_write_to_server_socket: + # self._data_to_write_to_server_socket.append([(data, addr), 0]) + #else: + self._server_socket.sendto(data, addr) + data = None + while self._data_to_write_to_server_socket: + data_buf = self._data_to_write_to_server_socket[0] + retry = data_buf[1] + 1 + del self._data_to_write_to_server_socket[0] + data, addr = data_buf[0] + self._server_socket.sendto(data, addr) + #""" + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + uncomplete = True + if error_no in (errno.EWOULDBLOCK,): + pass + else: + shell.print_exception(e) + return False + #if uncomplete and data is not None and retry < 3: + # self._data_to_write_to_server_socket.append([(data, addr), retry]) + #''' + def add_to_loop(self, loop): if self._eventloop: raise Exception('already add to loop') if self._closed: raise Exception('already closed') self._eventloop = loop - loop.add_handler(self._handle_events) server_socket = self._server_socket self._eventloop.add(server_socket, - eventloop.POLL_IN | eventloop.POLL_ERR) - - def _handle_events(self, events): - for sock, fd, event in events: - if sock == self._server_socket: - if event & eventloop.POLL_ERR: - logging.error('UDP server_socket err') - self._handle_server() - elif sock and (fd in self._sockets): - if event & eventloop.POLL_ERR: - logging.error('UDP client_socket err') - self._handle_client(sock) - now = time.time() - if now - self._last_time > 3: - self._cache.sweep() - self._client_fd_to_server_addr.sweep() - self._last_time = now + eventloop.POLL_IN | eventloop.POLL_ERR, self) + loop.add_periodic(self.handle_periodic) + + def remove_handler(self, handler): + index = self._handler_to_timeouts.get(hash(handler), -1) + if index >= 0: + # delete is O(n), so we just set it to None + self._timeouts[index] = None + del self._handler_to_timeouts[hash(handler)] + + def update_activity(self, handler): + # set handler to active + now = int(time.time()) + if now - handler.last_activity < eventloop.TIMEOUT_PRECISION: + # thus we can lower timeout modification frequency + return + handler.last_activity = now + index = self._handler_to_timeouts.get(hash(handler), -1) + if index >= 0: + # delete is O(n), so we just set it to None + self._timeouts[index] = None + length = len(self._timeouts) + self._timeouts.append(handler) + self._handler_to_timeouts[hash(handler)] = length + + def _sweep_timeout(self): + # tornado's timeout memory management is more flexible than we need + # we just need a sorted last_activity queue and it's faster than heapq + # in fact we can do O(1) insertion/remove so we invent our own + if self._timeouts: + logging.log(shell.VERBOSE_LEVEL, 'sweeping timeouts') + now = time.time() + length = len(self._timeouts) + pos = self._timeout_offset + while pos < length: + handler = self._timeouts[pos] + if handler: + if now - handler.last_activity < self._timeout: + break + else: + if handler.remote_address: + logging.warn('timed out: %s:%d' % + handler.remote_address) + else: + logging.warn('timed out') + handler.destroy() + handler.destroy_local() + self._timeouts[pos] = None # free memory + pos += 1 + else: + pos += 1 + if pos > TIMEOUTS_CLEAN_SIZE and pos > length >> 1: + # clean up the timeout queue when it gets larger than half + # of the queue + self._timeouts = self._timeouts[pos:] + for key in self._handler_to_timeouts: + self._handler_to_timeouts[key] -= pos + pos = 0 + self._timeout_offset = pos + + def handle_event(self, sock, fd, event): + if sock == self._server_socket: + if event & eventloop.POLL_ERR: + logging.error('UDP server_socket err') + self._handle_server() + elif sock and (fd in self._sockets): + if event & eventloop.POLL_ERR: + logging.error('UDP client_socket err') + self._handle_client(sock) + else: + if sock: + handler = self._fd_to_handlers.get(fd, None) + if handler: + handler.handle_event(sock, event) + else: + logging.warn('poll removed fd') + + def handle_periodic(self): if self._closed: - self._server_socket.close() - for sock in self._sockets: - sock.close() - self._eventloop.remove_handler(self._handle_events) + if self._server_socket: + self._server_socket.close() + self._server_socket = None + for sock in self._sockets: + sock.close() + logging.info('closed UDP port %d', self._listen_port) + self._cache.sweep() + self._client_fd_to_server_addr.sweep() + self._sweep_timeout() def close(self, next_tick=False): + logging.debug('UDP close') self._closed = True if not next_tick: + if self._eventloop: + self._eventloop.remove_periodic(self.handle_periodic) + self._eventloop.remove(self._server_socket) self._server_socket.close() From 6e5744829ce51bf04e32d4798394de5e8b4c49b0 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Thu, 6 Aug 2015 12:26:45 +0800 Subject: [PATCH 024/487] fix asyncmgr.py --- asyncmgr.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/asyncmgr.py b/asyncmgr.py index 8e24b421..8fc87a00 100644 --- a/asyncmgr.py +++ b/asyncmgr.py @@ -55,8 +55,7 @@ def add_to_loop(self, loop): socket.SOL_UDP) self._sock.bind((Config.MANAGE_BIND_IP, Config.MANAGE_PORT)) self._sock.setblocking(False) - loop.add(self._sock, eventloop.POLL_IN) - loop.add_handler(self.handle_events) + loop.add(self._sock, eventloop.POLL_IN, self) def _handle_data(self, sock): data, addr = sock.recvfrom(128) @@ -70,25 +69,25 @@ def _handle_data(self, sock): elif args[3] == '1': server_pool.ServerPool.get_instance().new_server(args[1], args[2]) - def handle_events(self, events): - for sock, fd, event in events: - if sock != self._sock: - continue - if event & eventloop.POLL_ERR: - logging.error('mgr socket err') - self._loop.remove(self._sock) - self._sock.close() - # TODO when dns server is IPv6 - self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, - socket.SOL_UDP) - self._sock.setblocking(False) - self._loop.add(self._sock, eventloop.POLL_IN) - else: - self._handle_data(sock) - break + def handle_events(self, sock, event): + if sock != self._sock: + return + if event & eventloop.POLL_ERR: + logging.error('mgr socket err') + self._loop.remove(self._sock) + self._sock.close() + # TODO when dns server is IPv6 + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + self._sock.setblocking(False) + self._loop.add(self._sock, eventloop.POLL_IN, self) + else: + self._handle_data(sock) def close(self): if self._sock: + if self._loop: + self._loop.remove(self._sock) self._sock.close() self._sock = None From e5e3102591c0016cdb114aecb19ae5a43b1d1b44 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Mon, 10 Aug 2015 12:47:37 +0800 Subject: [PATCH 025/487] remove debug code add AddressMap for udp --- shadowsocks/udprelay.py | 80 +++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 018a6a6d..34608b10 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -97,8 +97,9 @@ BUF_SIZE = 65536 DOUBLE_SEND_BEG_IDS = 16 -POST_MTU_MIN = 1000 +POST_MTU_MIN = 500 POST_MTU_MAX = 1400 +SENDING_WINDOW_SIZE = 8192 STAGE_INIT = 0 STAGE_RSP_ID = 1 @@ -173,9 +174,6 @@ def set_finish(self, begin_id, done_list): while self.begin_id < begin_id: self.begin_id += 1 del self.queue[self.begin_id] - #while len(self.queue) > 0 and self.queue[0][0] <= begin_id: - # del self.queue[0] - # self.begin_id += 1 class RecvQueue(object): def __init__(self): @@ -229,6 +227,38 @@ def get_missing_id(self, begin_id): missing.append(i - begin_id) return (begin_id, missing) +class AddressMap(object): + def __init__(self): + self._queue = [] + self._addr_map = {} + + def add(self, addr): + if addr in self._addr_map: + self._addr_map[addr] = UDPLocalAddress(addr) + else: + self._addr_map[addr] = UDPLocalAddress(addr) + self._queue.append(addr) + + def keys(self): + return self._queue + + def get(self): + if self._queue: + while True: + if len(self._queue) == 1: + return self._queue[0] + index = random.randint(0, len(self._queue) - 1) + addr = self._queue[index] + if self._addr_map[addr].is_timeout(): + self._queue[index] = self._queue[len(self._queue) - 1] + del self._queue[len(self._queue) - 1] + del self._addr_map[addr] + else: + break + return addr + else: + return None + class TCPRelayHandler(object): def __init__(self, server, reqid_to_handlers, fd_to_handlers, loop, local_sock, local_id, client_param, config, @@ -254,7 +284,7 @@ def __init__(self, server, reqid_to_handlers, fd_to_handlers, loop, self._upstream_status = WAIT_STATUS_READING self._downstream_status = WAIT_STATUS_INIT self._request_id = 0 - self._client_address = {} + self._client_address = AddressMap() self._remote_address = None self._sendingqueue = SendingQueue() self._recvqueue = RecvQueue() @@ -282,7 +312,10 @@ def remote_address(self): return self._remote_address def add_local_address(self, addr): - self._client_address[addr] = UDPLocalAddress(addr) + self._client_address.add(addr) + + def get_local_address(self): + return self._client_address.get() def _update_activity(self): # tell the TCP Relay we have activities recently @@ -367,8 +400,6 @@ def _write_to_sock(self, data, sock, addr = None): return False if uncomplete: if sock == self._local_sock: - #if data is not None and retry < 10: - # self._data_to_write_to_local.append([(data, addr), retry]) self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) elif sock == self._remote_sock: self._data_to_write_to_remote.append(data) @@ -377,15 +408,12 @@ def _write_to_sock(self, data, sock, addr = None): logging.error('write_all_to_sock:unknown socket') else: if sock == self._local_sock: - if self._sendingqueue.size() > 8192: + if self._sendingqueue.size() > SENDING_WINDOW_SIZE: self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) else: self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) elif sock == self._remote_sock: - if self._sendingqueue.size() > 8192: - self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) - else: - self._update_stream(STREAM_UP, WAIT_STATUS_READING) + self._update_stream(STREAM_UP, WAIT_STATUS_READING) else: logging.error('write_all_to_sock:unknown socket') return True @@ -439,9 +467,7 @@ def _handle_dns_resolved(self, result, error): self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) self._stage = STAGE_STREAM - for it_addr in self._client_address: - addr = it_addr - break + addr = self.get_local_address() for i in xrange(2): rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, "\x02") @@ -508,13 +534,11 @@ def _on_remote_read(self): pack_id = self._sendingqueue.append(data) post_data = self._pack_post_data(CMD_POST, pack_id, data) - for it_addr in self._client_address: - addr = it_addr - break + addr = self.get_local_address() self._write_to_sock(post_data, self._local_sock, addr) - #if pack_id <= DOUBLE_SEND_BEG_IDS: - # post_data = self._pack_post_data(CMD_POST, pack_id, data) - # self._write_to_sock(post_data, self._local_sock, addr) + if pack_id <= DOUBLE_SEND_BEG_IDS: + post_data = self._pack_post_data(CMD_POST, pack_id, data) + self._write_to_sock(post_data, self._local_sock, addr) except Exception as e: shell.print_exception(e) @@ -620,9 +644,9 @@ def handle_stream_sync_status(self, addr, cmd, request_id, pack_id, max_send_id, for post_pack_id, post_data in send_list: rsp_data = self._pack_post_data(CMD_POST, post_pack_id, post_data) self._write_to_sock(rsp_data, self._local_sock, addr) - #if post_pack_id <= DOUBLE_SEND_BEG_IDS: - # rsp_data = self._pack_post_data(CMD_POST, post_pack_id, post_data) - # self._write_to_sock(rsp_data, self._local_sock, addr) + if post_pack_id <= DOUBLE_SEND_BEG_IDS: + rsp_data = self._pack_post_data(CMD_POST, post_pack_id, post_data) + self._write_to_sock(rsp_data, self._local_sock, addr) def handle_client(self, addr, cmd, request_id, data): self.add_local_address(addr) @@ -732,13 +756,11 @@ def handle_client(self, addr, cmd, request_id, data): pack_id = struct.unpack(">I", data[0:4])[0] max_send_id = struct.unpack(">I", data[4:8])[0] data = data[8:] - logging.info('handle_client STAGE_DESTROYED send %d %d' % (request_id, pack_id)) self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) elif cmd == CMD_SYN_STATUS_64: pack_id = struct.unpack(">Q", data[0:8])[0] max_send_id = struct.unpack(">Q", data[8:16])[0] data = data[16:] - logging.info('handle_client STAGE_DESTROYED send %d %d' % (request_id, pack_id)) self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) def handle_event(self, sock, event): @@ -810,9 +832,7 @@ def destroy_local(self): logging.debug('disconnect local') rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") addr = None - for it_addr in self._client_address: - addr = it_addr - break + addr = self.get_local_address() self._write_to_sock(rsp_data, self._local_sock, addr) self._local_sock = None del self._reqid_to_handlers[self._request_id] From a3344d9f1c5b88184778986ef324481fcaa41204 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 11 Aug 2015 11:34:01 +0800 Subject: [PATCH 026/487] rename handle_event --- asyncmgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asyncmgr.py b/asyncmgr.py index 8fc87a00..3d5397f9 100644 --- a/asyncmgr.py +++ b/asyncmgr.py @@ -69,7 +69,7 @@ def _handle_data(self, sock): elif args[3] == '1': server_pool.ServerPool.get_instance().new_server(args[1], args[2]) - def handle_events(self, sock, event): + def handle_event(self, sock, event): if sock != self._sock: return if event & eventloop.POLL_ERR: From 661fbbfb8b6c1888f760a57bbeee81ae616a9b2a Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 12 Aug 2015 01:10:28 +0800 Subject: [PATCH 027/487] fix handle_event --- asyncmgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asyncmgr.py b/asyncmgr.py index 3d5397f9..9bf4d093 100644 --- a/asyncmgr.py +++ b/asyncmgr.py @@ -69,7 +69,7 @@ def _handle_data(self, sock): elif args[3] == '1': server_pool.ServerPool.get_instance().new_server(args[1], args[2]) - def handle_event(self, sock, event): + def handle_event(self, sock, fd, event): if sock != self._sock: return if event & eventloop.POLL_ERR: From b32cc8f703996835146c0063eede289bad0d9dc6 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 12 Aug 2015 12:32:20 +0800 Subject: [PATCH 028/487] fix closing relays --- db_transfer.py | 6 +++--- server_pool.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 65665273..e374855b 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -118,12 +118,12 @@ def del_server_out_of_bound_safe(last_rows, rows): if ServerPool.get_instance().server_is_run(port) > 0: if not allow: logging.info('db stop server at port [%s]' % (port,)) - ServerPool.get_instance().del_server(port) + ServerPool.get_instance().cb_del_server(port) elif (port in ServerPool.get_instance().tcp_servers_pool and ServerPool.get_instance().tcp_servers_pool[port]._config['password'] != passwd) \ or (port in ServerPool.get_instance().tcp_ipv6_servers_pool and ServerPool.get_instance().tcp_ipv6_servers_pool[port]._config['password'] != passwd): #password changed logging.info('db stop server at port [%s] reason: password changed' % (port,)) - ServerPool.get_instance().del_server(port) + ServerPool.get_instance().cb_del_server(port) elif allow and ServerPool.get_instance().server_run_status(port) is False: logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) @@ -134,7 +134,7 @@ def del_server_out_of_bound_safe(last_rows, rows): pass else: logging.info('db stop server at port [%s] reason: port not exist' % (row['port'])) - ServerPool.get_instance().del_server(row['port']) + ServerPool.get_instance().cb_del_server(row['port']) @staticmethod def thread_db(): diff --git a/server_pool.py b/server_pool.py index cdeba36a..ee0e86b1 100644 --- a/server_pool.py +++ b/server_pool.py @@ -167,12 +167,12 @@ def cb_del_server(self, port): else: logging.info("stopped server at %s:%d" % (self.config['server'], port)) try: - self.tcp_servers_pool[port].close(True) + self.tcp_servers_pool[port].close(False) del self.tcp_servers_pool[port] except Exception, e: logging.warn(e) try: - self.udp_servers_pool[port].close(True) + self.udp_servers_pool[port].close(False) del self.udp_servers_pool[port] except Exception, e: logging.warn(e) @@ -183,12 +183,12 @@ def cb_del_server(self, port): else: logging.info("stopped server at %s:%d" % (self.config['server_ipv6'], port)) try: - self.tcp_ipv6_servers_pool[port].close(True) + self.tcp_ipv6_servers_pool[port].close(False) del self.tcp_ipv6_servers_pool[port] except Exception, e: logging.warn(e) try: - self.udp_ipv6_servers_pool[port].close(True) + self.udp_ipv6_servers_pool[port].close(False) del self.udp_ipv6_servers_pool[port] except Exception, e: logging.warn(e) From da4686682851589b98d251506b3b33dcd5ec9011 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 12 Aug 2015 12:45:07 +0800 Subject: [PATCH 029/487] restart server quickly when change password --- db_transfer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/db_transfer.py b/db_transfer.py index e374855b..a62c30f8 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -124,6 +124,7 @@ def del_server_out_of_bound_safe(last_rows, rows): #password changed logging.info('db stop server at port [%s] reason: password changed' % (port,)) ServerPool.get_instance().cb_del_server(port) + ServerPool.get_instance().new_server(port, passwd) elif allow and ServerPool.get_instance().server_run_status(port) is False: logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) From 9defba2d36c5ea6d0ea4652add220a119cdefb9b Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 12 Aug 2015 14:37:17 +0800 Subject: [PATCH 030/487] merge master 20150810 --- shadowsocks/eventloop.py | 9 ++ shadowsocks/manager.py | 286 +++++++++++++++++++++++++++++++++++++++ shadowsocks/server.py | 12 +- shadowsocks/shell.py | 13 +- shadowsocks/tcprelay.py | 25 ++-- shadowsocks/udprelay.py | 58 ++++---- 6 files changed, 363 insertions(+), 40 deletions(-) create mode 100644 shadowsocks/manager.py diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index b27afe30..ce9c11bc 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -98,6 +98,9 @@ def modify(self, fd, mode): self.unregister(fd) self.register(fd, mode) + def close(self): + self.kqueue.close() + class SelectLoop(object): @@ -135,6 +138,9 @@ def modify(self, fd, mode): self.unregister(fd) self.register(fd, mode) + def close(self): + pass + class EventLoop(object): def __init__(self): @@ -216,6 +222,9 @@ def run(self): callback() self._last_time = now + def __del__(self): + self._impl.close() + # from tornado def errno_from_exception(e): diff --git a/shadowsocks/manager.py b/shadowsocks/manager.py new file mode 100644 index 00000000..e8009b48 --- /dev/null +++ b/shadowsocks/manager.py @@ -0,0 +1,286 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright 2015 clowwindy +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import errno +import traceback +import socket +import logging +import json +import collections + +from shadowsocks import common, eventloop, tcprelay, udprelay, asyncdns, shell + + +BUF_SIZE = 1506 +STAT_SEND_LIMIT = 100 + + +class Manager(object): + + def __init__(self, config): + self._config = config + self._relays = {} # (tcprelay, udprelay) + self._loop = eventloop.EventLoop() + self._dns_resolver = asyncdns.DNSResolver() + self._dns_resolver.add_to_loop(self._loop) + + self._statistics = collections.defaultdict(int) + self._control_client_addr = None + try: + manager_address = config['manager_address'] + if ':' in manager_address: + addr = manager_address.rsplit(':', 1) + addr = addr[0], int(addr[1]) + addrs = socket.getaddrinfo(addr[0], addr[1]) + if addrs: + family = addrs[0][0] + else: + logging.error('invalid address: %s', manager_address) + exit(1) + else: + addr = manager_address + family = socket.AF_UNIX + self._control_socket = socket.socket(family, + socket.SOCK_DGRAM) + self._control_socket.bind(addr) + self._control_socket.setblocking(False) + except (OSError, IOError) as e: + logging.error(e) + logging.error('can not bind to manager address') + exit(1) + self._loop.add(self._control_socket, + eventloop.POLL_IN, self) + self._loop.add_periodic(self.handle_periodic) + + port_password = config['port_password'] + del config['port_password'] + for port, password in port_password.items(): + a_config = config.copy() + a_config['server_port'] = int(port) + a_config['password'] = password + self.add_port(a_config) + + def add_port(self, config): + port = int(config['server_port']) + servers = self._relays.get(port, None) + if servers: + logging.error("server already exists at %s:%d" % (config['server'], + port)) + return + logging.info("adding server at %s:%d" % (config['server'], port)) + t = tcprelay.TCPRelay(config, self._dns_resolver, False, + self.stat_callback) + u = udprelay.UDPRelay(config, self._dns_resolver, False, + self.stat_callback) + t.add_to_loop(self._loop) + u.add_to_loop(self._loop) + self._relays[port] = (t, u) + + def remove_port(self, config): + port = int(config['server_port']) + servers = self._relays.get(port, None) + if servers: + logging.info("removing server at %s:%d" % (config['server'], port)) + t, u = servers + t.close(next_tick=False) + u.close(next_tick=False) + del self._relays[port] + else: + logging.error("server not exist at %s:%d" % (config['server'], + port)) + + def handle_event(self, sock, fd, event): + if sock == self._control_socket and event == eventloop.POLL_IN: + data, self._control_client_addr = sock.recvfrom(BUF_SIZE) + parsed = self._parse_command(data) + if parsed: + command, config = parsed + a_config = self._config.copy() + if config: + # let the command override the configuration file + a_config.update(config) + if 'server_port' not in a_config: + logging.error('can not find server_port in config') + else: + if command == 'add': + self.add_port(a_config) + self._send_control_data(b'ok') + elif command == 'remove': + self.remove_port(a_config) + self._send_control_data(b'ok') + elif command == 'ping': + self._send_control_data(b'pong') + else: + logging.error('unknown command %s', command) + + def _parse_command(self, data): + # commands: + # add: {"server_port": 8000, "password": "foobar"} + # remove: {"server_port": 8000"} + data = common.to_str(data) + parts = data.split(':', 1) + if len(parts) < 2: + return data, None + command, config_json = parts + try: + config = shell.parse_json_in_str(config_json) + return command, config + except Exception as e: + logging.error(e) + return None + + def stat_callback(self, port, data_len): + self._statistics[port] += data_len + + def handle_periodic(self): + r = {} + i = 0 + + def send_data(data_dict): + if data_dict: + # use compact JSON format (without space) + data = common.to_bytes(json.dumps(data_dict, + separators=(',', ':'))) + self._send_control_data(b'stat: ' + data) + + for k, v in self._statistics.items(): + r[k] = v + i += 1 + # split the data into segments that fit in UDP packets + if i >= STAT_SEND_LIMIT: + send_data(r) + r.clear() + send_data(r) + self._statistics.clear() + + def _send_control_data(self, data): + if self._control_client_addr: + try: + self._control_socket.sendto(data, self._control_client_addr) + except (socket.error, OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): + return + else: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() + + def run(self): + self._loop.run() + + +def run(config): + Manager(config).run() + + +def test(): + import time + import threading + import struct + from shadowsocks import encrypt + + logging.basicConfig(level=5, + format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + enc = [] + eventloop.TIMEOUT_PRECISION = 1 + + def run_server(): + config = { + 'server': '127.0.0.1', + 'local_port': 1081, + 'port_password': { + '8381': 'foobar1', + '8382': 'foobar2' + }, + 'method': 'aes-256-cfb', + 'manager_address': '127.0.0.1:6001', + 'timeout': 60, + 'fast_open': False, + 'verbose': 2 + } + manager = Manager(config) + enc.append(manager) + manager.run() + + t = threading.Thread(target=run_server) + t.start() + time.sleep(1) + manager = enc[0] + cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + cli.connect(('127.0.0.1', 6001)) + + # test add and remove + time.sleep(1) + cli.send(b'add: {"server_port":7001, "password":"asdfadsfasdf"}') + time.sleep(1) + assert 7001 in manager._relays + data, addr = cli.recvfrom(1506) + assert b'ok' in data + + cli.send(b'remove: {"server_port":8381}') + time.sleep(1) + assert 8381 not in manager._relays + data, addr = cli.recvfrom(1506) + assert b'ok' in data + logging.info('add and remove test passed') + + # test statistics for TCP + header = common.pack_addr(b'google.com') + struct.pack('>H', 80) + data = encrypt.encrypt_all(b'asdfadsfasdf', 'aes-256-cfb', 1, + header + b'GET /\r\n\r\n') + tcp_cli = socket.socket() + tcp_cli.connect(('127.0.0.1', 7001)) + tcp_cli.send(data) + tcp_cli.recv(4096) + tcp_cli.close() + + data, addr = cli.recvfrom(1506) + data = common.to_str(data) + assert data.startswith('stat: ') + data = data.split('stat:')[1] + stats = shell.parse_json_in_str(data) + assert '7001' in stats + logging.info('TCP statistics test passed') + + # test statistics for UDP + header = common.pack_addr(b'127.0.0.1') + struct.pack('>H', 80) + data = encrypt.encrypt_all(b'foobar2', 'aes-256-cfb', 1, + header + b'test') + udp_cli = socket.socket(type=socket.SOCK_DGRAM) + udp_cli.sendto(data, ('127.0.0.1', 8382)) + tcp_cli.close() + + data, addr = cli.recvfrom(1506) + data = common.to_str(data) + assert data.startswith('stat: ') + data = data.split('stat:')[1] + stats = json.loads(data) + assert '8382' in stats + logging.info('UDP statistics test passed') + + manager._loop.stop() + t.join() + + +if __name__ == '__main__': + test() diff --git a/shadowsocks/server.py b/shadowsocks/server.py index d919092a..68a6716b 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -24,7 +24,8 @@ import signal sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) -from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, asyncdns +from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, \ + asyncdns, manager def main(): @@ -48,10 +49,17 @@ def main(): else: config['port_password'][str(server_port)] = config['password'] + if config.get('manager_address', 0): + logging.info('entering manager mode') + manager.run(config) + return + tcp_servers = [] udp_servers = [] dns_resolver = asyncdns.DNSResolver() - for port, password in config['port_password'].items(): + port_password = config['port_password'] + del config['port_password'] + for port, password in port_password.items(): a_config = config.copy() ipv6_ok = False logging.info("server start with password [%s] method [%s]" % (password, a_config['method'])) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index f8ae81f5..c91fc222 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -136,7 +136,7 @@ def get_config(is_local): else: shortopts = 'hd:s:p:k:m:c:t:vq' longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'workers=', - 'forbidden-ip=', 'user=', 'version'] + 'forbidden-ip=', 'user=', 'manager-address=', 'version'] try: config_path = find_config() optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) @@ -148,8 +148,7 @@ def get_config(is_local): logging.info('loading config from %s' % config_path) with open(config_path, 'rb') as f: try: - config = json.loads(f.read().decode('utf8'), - object_hook=_decode_dict) + config = parse_json_in_str(f.read().decode('utf8')) except ValueError as e: logging.error('found an error in config.json: %s', e.message) @@ -181,6 +180,8 @@ def get_config(is_local): config['fast_open'] = True elif key == '--workers': config['workers'] = int(value) + elif key == '--manager-address': + config['manager_address'] = value elif key == '--user': config['user'] = to_str(value) elif key == '--forbidden-ip': @@ -317,6 +318,7 @@ def print_server_help(): --fast-open use TCP_FASTOPEN, requires Linux 3.7+ --workers WORKERS number of workers, available on Unix/Linux --forbidden-ip IPLIST comma seperated IP list forbidden to connect + --manager-address ADDR optional server manager UDP address, see wiki General options: -h, --help show this help message and exit @@ -356,3 +358,8 @@ def _decode_dict(data): value = _decode_dict(value) rv[key] = value return rv + + +def parse_json_in_str(data): + # parse json and convert everything from unicode to str + return json.loads(data, object_hook=_decode_dict) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8188a005..e9d78e22 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -23,6 +23,7 @@ import errno import struct import logging +import binascii import traceback import random @@ -32,9 +33,6 @@ # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 -# we check timeouts every TIMEOUT_PRECISION seconds -TIMEOUT_PRECISION = 4 - MSG_FASTOPEN = 0x20000000 # SOCKS command definition @@ -153,10 +151,10 @@ def _get_a_server(self): logging.debug('chosen server: %s:%d', server, server_port) return server, server_port - def _update_activity(self): + def _update_activity(self, data_len=0): # tell the TCP Relay we have activities recently # else it will think we are inactive and timed out - self._server.update_activity(self) + self._server.update_activity(self, data_len) def _update_stream(self, stream, status): # update a stream to a new waiting status @@ -379,7 +377,6 @@ def _handle_stage_addr(self, data): self._log_error(e) if self._config['verbose']: traceback.print_exc() - # TODO use logging when debug completed self.destroy() def _create_remote_socket(self, ip, port): @@ -397,7 +394,6 @@ def _create_remote_socket(self, ip, port): common.to_str(sa[0])) remote_sock = socket.socket(af, socktype, proto) self._remote_sock = remote_sock - self._fd_to_handlers[remote_sock.fileno()] = self if self._remote_udp: @@ -410,7 +406,6 @@ def _create_remote_socket(self, ip, port): remote_sock_v6.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) remote_sock_v6.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) - remote_sock.setblocking(False) if self._remote_udp: pass @@ -483,7 +478,6 @@ def _handle_dns_resolved(self, result, error): def _on_local_read(self): # handle all local read events and dispatch them to methods for # each stage - self._update_activity() if not self._local_sock: return is_local = self._is_local @@ -497,6 +491,7 @@ def _on_local_read(self): if not data: self.destroy() return + self._update_activity(len(data)) if not is_local: data = self._encryptor.decrypt(data) if not data: @@ -520,7 +515,6 @@ def _on_local_read(self): def _on_remote_read(self, is_remote_sock): # handle all remote read events - self._update_activity() data = None try: if self._remote_udp: @@ -547,6 +541,7 @@ def _on_remote_read(self, is_remote_sock): self.destroy() return self._server.server_transfer_dl += len(data) + self._update_activity(len(data)) if self._is_local: data = self._encryptor.decrypt(data) else: @@ -667,7 +662,7 @@ def destroy(self): class TCPRelay(object): - def __init__(self, config, dns_resolver, is_local): + def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._config = config self._is_local = is_local self._dns_resolver = dns_resolver @@ -709,6 +704,7 @@ def __init__(self, config, dns_resolver, is_local): self._config['fast_open'] = False server_socket.listen(1024) self._server_socket = server_socket + self._stat_callback = stat_callback def add_to_loop(self, loop): if self._eventloop: @@ -727,7 +723,10 @@ def remove_handler(self, handler): self._timeouts[index] = None del self._handler_to_timeouts[hash(handler)] - def update_activity(self, handler): + def update_activity(self, handler, data_len): + if data_len and self._stat_callback: + self._stat_callback(self._listen_port, data_len) + # set handler to active now = int(time.time()) if now - handler.last_activity < eventloop.TIMEOUT_PRECISION: @@ -828,3 +827,5 @@ def close(self, next_tick=False): self._eventloop.remove_periodic(self.handle_periodic) self._eventloop.remove(self._server_socket) self._server_socket.close() + for handler in list(self._fd_to_handlers.values()): + handler.destroy() diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 34608b10..4c5ff219 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -77,9 +77,6 @@ # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 -# we check timeouts every TIMEOUT_PRECISION seconds -TIMEOUT_PRECISION = 4 - # for each handler, we have 2 stream directions: # upstream: from client to server direction # read local and write to remote @@ -120,6 +117,14 @@ CMD_VER_STR = "\x08" +RSP_STATE_EMPTY = "" +RSP_STATE_REJECT = "\x00" +RSP_STATE_CONNECTED = "\x01" +RSP_STATE_CONNECTEDREMOTE = "\x02" +RSP_STATE_ERROR = "\x03" +RSP_STATE_DISCONNECT = "\x04" +RSP_STATE_REDIRECT = "\x05" + class UDPLocalAddress(object): def __init__(self, addr): self.addr = addr @@ -470,7 +475,7 @@ def _handle_dns_resolved(self, result, error): addr = self.get_local_address() for i in xrange(2): - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, "\x02") + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) self._write_to_sock(rsp_data, self._local_sock, addr) return @@ -651,7 +656,7 @@ def handle_stream_sync_status(self, addr, cmd, request_id, pack_id, max_send_id, def handle_client(self, addr, cmd, request_id, data): self.add_local_address(addr) if cmd == CMD_DISCONNECT: - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) self._write_to_sock(rsp_data, self._local_sock, addr) self.destroy() self.destroy_local() @@ -667,7 +672,7 @@ def handle_client(self, addr, cmd, request_id, data): if self._stage == STAGE_RSP_ID: if cmd == CMD_CONNECT: for i in xrange(2): - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, "\x01") + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, RSP_STATE_CONNECTED) self._write_to_sock(rsp_data, self._local_sock, addr) elif cmd == CMD_CONNECT_REMOTE: local_id = data[0:4] @@ -684,35 +689,35 @@ def handle_client(self, addr, cmd, request_id, data): logging.info('TCP connect %s:%d from %s:%d' % (remote_addr, remote_port, addr[0], addr[1])) else: # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) self._write_to_sock(rsp_data, self._local_sock, addr) elif self._stage == STAGE_CONNECTING: if cmd == CMD_CONNECT_REMOTE: local_id = data[0:4] if self._local_id == local_id: for i in xrange(2): - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, "\x02") + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) self._write_to_sock(rsp_data, self._local_sock, addr) else: # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) self._write_to_sock(rsp_data, self._local_sock, addr) elif self._stage == STAGE_STREAM: if len(data) < 4: # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) self._write_to_sock(rsp_data, self._local_sock, addr) return local_id = data[0:4] if self._local_id != local_id: # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) self._write_to_sock(rsp_data, self._local_sock, addr) return else: data = data[4:] if cmd == CMD_CONNECT_REMOTE: - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, "\x02") + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) self._write_to_sock(rsp_data, self._local_sock, addr) elif cmd == CMD_POST: recv_id = struct.unpack(">I", data[0:4])[0] @@ -725,7 +730,7 @@ def handle_client(self, addr, cmd, request_id, data): self._recvqueue.insert(pack_id, data[16:]) self._sendingqueue.set_finish(recv_id, []) elif cmd == CMD_DISCONNECT: - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) self._write_to_sock(rsp_data, self._local_sock, addr) self.destroy() self.destroy_local() @@ -747,7 +752,7 @@ def handle_client(self, addr, cmd, request_id, data): local_id = data[0:4] if self._local_id != local_id: # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) self._write_to_sock(rsp_data, self._local_sock, addr) return else: @@ -830,7 +835,7 @@ def destroy(self): def destroy_local(self): if self._local_sock: logging.debug('disconnect local') - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) addr = None addr = self.get_local_address() self._write_to_sock(rsp_data, self._local_sock, addr) @@ -842,8 +847,9 @@ def client_key(source_addr, server_af): # notice this is server af, not dest af return '%s:%s:%d' % (source_addr[0], source_addr[1], server_af) + class UDPRelay(object): - def __init__(self, config, dns_resolver, is_local): + def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._config = config if is_local: self._listen_addr = config['local_address'] @@ -856,7 +862,7 @@ def __init__(self, config, dns_resolver, is_local): self._remote_addr = None self._remote_port = None self._dns_resolver = dns_resolver - self._password = config['password'] + self._password = common.to_bytes(config['password']) self._method = config['method'] self._timeout = config['timeout'] self._is_local = is_local @@ -897,6 +903,7 @@ def __init__(self, config, dns_resolver, is_local): server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) self._server_socket = server_socket + self._stat_callback = stat_callback def _get_a_server(self): server = self._config['server'] @@ -957,6 +964,8 @@ def _handle_server(self): data, r_addr = server.recvfrom(BUF_SIZE) if not data: logging.debug('UDP handle_server: data is empty') + if self._stat_callback: + self._stat_callback(self._listen_port, len(data)) if self._is_local: frag = common.ord(data[2]) if frag != 0: @@ -996,7 +1005,7 @@ def _handle_server(self): break # return req id self._reqid_to_hd[req_id] = (data[2][0:4], None) - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, req_id, "\x01") + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, req_id, RSP_STATE_CONNECTED) data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) self.write_to_server_socket(data_to_send, r_addr) elif data[0] == CMD_CONNECT_REMOTE: @@ -1014,7 +1023,7 @@ def _handle_server(self): self.update_activity(handle) else: # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) self.write_to_server_socket(data_to_send, r_addr) else: @@ -1022,7 +1031,7 @@ def _handle_server(self): self._reqid_to_hd[data[1]].handle_client(r_addr, *data) else: # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) self.write_to_server_socket(data_to_send, r_addr) elif data[0] > CMD_CONNECT_REMOTE and data[0] <= CMD_DISCONNECT: @@ -1031,7 +1040,7 @@ def _handle_server(self): self._reqid_to_hd[data[1]].handle_client(r_addr, *data) else: # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], "") + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) self.write_to_server_socket(data_to_send, r_addr) return @@ -1062,7 +1071,6 @@ def _handle_server(self): af, socktype, proto, canonname, sa = addrs[0] key = client_key(r_addr, af) - logging.debug(key) client = self._cache.get(key, None) if not client: # TODO async getaddrinfo @@ -1103,6 +1111,8 @@ def _handle_client(self, sock): if not data: logging.debug('UDP handle_client: data is empty') return + if self._stat_callback: + self._stat_callback(self._listen_port, len(data)) if not self._is_local: addrlen = len(r_addr[0]) if addrlen > 255: @@ -1121,7 +1131,7 @@ def _handle_client(self, sock): header_result = parse_header(data) if header_result is None: return - connecttype, dest_addr, dest_port, header_length = header_result + #connecttype, dest_addr, dest_port, header_length = header_result #logging.debug('UDP handle_client %s:%d to %s:%d' % (common.to_str(r_addr[0]), r_addr[1], dest_addr, dest_port)) response = b'\x00\x00\x00' + data @@ -1270,3 +1280,5 @@ def close(self, next_tick=False): self._eventloop.remove_periodic(self.handle_periodic) self._eventloop.remove(self._server_socket) self._server_socket.close() + for client in list(self._cache.values()): + client.close() From 928c26e4f2d3bff7a4ff750f472a8959efc05be1 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Sat, 15 Aug 2015 12:47:14 +0800 Subject: [PATCH 031/487] fix some exceptions --- shadowsocks/encrypt.py | 6 +++++- shadowsocks/udprelay.py | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 834b18ce..5d111380 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -21,6 +21,7 @@ import sys import hashlib import logging +import random from shadowsocks import common from shadowsocks.crypto import rc4_md5, openssl, sodium, table @@ -34,7 +35,10 @@ def random_string(length): - return os.urandom(length) + try: + return os.urandom(length) + except (AttributeError, NotImplementedError): + return ''.join(chr(random.randrange(255)) for _ in range(length)) cached_keys = {} diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 4c5ff219..3ea9b6d8 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1036,8 +1036,11 @@ def _handle_server(self): self.write_to_server_socket(data_to_send, r_addr) elif data[0] > CMD_CONNECT_REMOTE and data[0] <= CMD_DISCONNECT: if data[1] in self._reqid_to_hd: - self.update_activity(self._reqid_to_hd[data[1]]) - self._reqid_to_hd[data[1]].handle_client(r_addr, *data) + if type(self._reqid_to_hd[data[1]]) is tuple: + pass + else: + self.update_activity(self._reqid_to_hd[data[1]]) + self._reqid_to_hd[data[1]].handle_client(r_addr, *data) else: # disconnect rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) From 61959945c3591c062ba44dc8639cb32109fff446 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Thu, 27 Aug 2015 16:41:00 +0800 Subject: [PATCH 032/487] revert encrypt.py new obfs header --- shadowsocks/common.py | 21 ++++++++++++++++++--- shadowsocks/encrypt.py | 6 +----- shadowsocks/tcprelay.py | 2 ++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 11b06225..cc88d5d1 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -21,7 +21,7 @@ import socket import struct import logging - +import binascii def compat_ord(s): if type(s) == int: @@ -140,7 +140,7 @@ def pack_addr(address): def pre_parse_header(data): datatype = ord(data[0]) - if datatype == 0x80 : + if datatype == 0x80: if len(data) <= 2: return None rand_data_size = ord(data[1]) @@ -151,7 +151,7 @@ def pre_parse_header(data): data = data[rand_data_size + 2:] elif datatype == 0x81: data = data[1:] - elif datatype == 0x82 : + elif datatype == 0x82: if len(data) <= 3: return None rand_data_size = struct.unpack('>H', data[1:3])[0] @@ -160,6 +160,21 @@ def pre_parse_header(data): 'encryption method') return None data = data[rand_data_size + 3:] + elif datatype == 0x88: + if len(data) <= 7 + 7: + return None + data_size = struct.unpack('>H', data[1:3])[0] + ogn_data = data + data = data[:data_size] + crc = binascii.crc32(data) & 0xffffffff + if crc != 0xffffffff: + logging.warn('uncorrect CRC32, maybe wrong password or ' + 'encryption method') + return None + start_pos = 3 + ord(data[3]) + data = data[start_pos:-4] + if data_size < len(ogn_data): + data += ogn_data[data_size:] return data def parse_header(data): diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 5d111380..834b18ce 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -21,7 +21,6 @@ import sys import hashlib import logging -import random from shadowsocks import common from shadowsocks.crypto import rc4_md5, openssl, sodium, table @@ -35,10 +34,7 @@ def random_string(length): - try: - return os.urandom(length) - except (AttributeError, NotImplementedError): - return ''.join(chr(random.randrange(255)) for _ in range(length)) + return os.urandom(length) cached_keys = {} diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e9d78e22..8c531130 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -341,6 +341,8 @@ def _handle_stage_addr(self, data): logging.error('unknown command %d', cmd) self.destroy() return + if False and ord(data[0]) != 0x88: # force new header + raise Exception('can not parse header') data = pre_parse_header(data) if data is None: raise Exception('can not parse header') From 5249fd528cfaed4c9f04de6270e4365a0075387d Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 28 Aug 2015 13:35:10 +0800 Subject: [PATCH 033/487] add FORCE_NEW_PROTOCOL switch --- shadowsocks/tcprelay.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8c531130..cb478f47 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -30,6 +30,10 @@ from shadowsocks import encrypt, eventloop, shell, common from shadowsocks.common import pre_parse_header, parse_header +# set it 'False' to use both new protocol and the original shadowsocks protocal +# set it 'True' to use new protocol ONLY, to avoid GFW detecting +FORCE_NEW_PROTOCOL = False + # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 @@ -341,7 +345,7 @@ def _handle_stage_addr(self, data): logging.error('unknown command %d', cmd) self.destroy() return - if False and ord(data[0]) != 0x88: # force new header + if FORCE_NEW_PROTOCOL and ord(data[0]) != 0x88: raise Exception('can not parse header') data = pre_parse_header(data) if data is None: From 56af51b43c8d9d5bed39e2399fe426d0c05e167a Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 1 Sep 2015 16:58:05 +0800 Subject: [PATCH 034/487] log TCP header if parse error --- shadowsocks/tcprelay.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index cb478f47..314e96d0 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -319,7 +319,7 @@ def _handle_stage_connecting(self, data): traceback.print_exc() self.destroy() - def _handle_stage_addr(self, data): + def _handle_stage_addr(self, ogn_data, data): try: if self._is_local: cmd = common.ord(data[1]) @@ -345,13 +345,18 @@ def _handle_stage_addr(self, data): logging.error('unknown command %d', cmd) self.destroy() return + + before_parse_data = data if FORCE_NEW_PROTOCOL and ord(data[0]) != 0x88: + logging.warn("TCP data %s decrypt %s" % (binascii.hexlify(ogn_data), binascii.hexlify(before_parse_data))) raise Exception('can not parse header') data = pre_parse_header(data) if data is None: + logging.warn("TCP data %s decrypt %s" % (binascii.hexlify(ogn_data), binascii.hexlify(before_parse_data))) raise Exception('can not parse header') header_result = parse_header(data) if header_result is None: + logging.warn("TCP data %s decrypt %s" % (binascii.hexlify(ogn_data), binascii.hexlify(before_parse_data))) raise Exception('can not parse header') connecttype, remote_addr, remote_port, header_length = header_result logging.info('%s connecting %s:%d from %s:%d' % @@ -497,6 +502,7 @@ def _on_local_read(self): if not data: self.destroy() return + ogn_data = data self._update_activity(len(data)) if not is_local: data = self._encryptor.decrypt(data) @@ -517,7 +523,7 @@ def _on_local_read(self): self._handle_stage_connecting(data) elif (is_local and self._stage == STAGE_ADDR) or \ (not is_local and self._stage == STAGE_INIT): - self._handle_stage_addr(data) + self._handle_stage_addr(ogn_data, data) def _on_remote_read(self, is_remote_sock): # handle all remote read events From 55d10ace01eea22c2b9205c2e9a1eec6d4b12e79 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Wed, 2 Sep 2015 23:57:51 +0800 Subject: [PATCH 035/487] simple reverse proxy --- shadowsocks/tcprelay.py | 42 +++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 314e96d0..cecd0176 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -114,6 +114,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._stage = STAGE_INIT self._encryptor = encrypt.Encryptor(config['password'], config['method']) + self._encrypt_correct = True self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] @@ -281,6 +282,31 @@ def _write_to_sock(self, data, sock): logging.error('write_all_to_sock:unknown socket') return True + def _get_redirect_host(self, client_address, ogn_data): + # test + host_list = [("www.bing.com", 80), ("www.microsoft.com", 80), ("www.baidu.com", 443), ("www.qq.com", 80), ("www.csdn.net", 80), ("1.2.3.4", 1000)] + hash_code = binascii.crc32(ogn_data) + addrs = socket.getaddrinfo(client_address[0], client_address[1], 0, socket.SOCK_STREAM, socket.SOL_TCP) + af, socktype, proto, canonname, sa = addrs[0] + address_bytes = common.inet_pton(af, sa[0]) + if len(address_bytes) == 16: + addr = struct.unpack('>Q', address_bytes[8:])[0] + if len(address_bytes) == 4: + addr = struct.unpack('>I', address_bytes)[0] + else: + addr = 0 + return host_list[((hash_code & 0xffffffff) + addr + 3) % len(host_list)] + + def _handel_protocol_error(self, client_address, ogn_data): + logging.warn("Protocol ERROR, TCP ogn data %s" % (binascii.hexlify(ogn_data), )) + self._encrypt_correct = False + #create redirect or disconnect by hash code + host, port = self._get_redirect_host(client_address, ogn_data) + data = "\x03" + chr(len(host)) + host + struct.pack('>H', port) + logging.warn("TCP data redir %s:%d %s" % (host, port, binascii.hexlify(data))) + #raise Exception('can not parse header') + return data + ogn_data + def _handle_stage_connecting(self, data): if self._is_local: data = self._encryptor.encrypt(data) @@ -348,16 +374,14 @@ def _handle_stage_addr(self, ogn_data, data): before_parse_data = data if FORCE_NEW_PROTOCOL and ord(data[0]) != 0x88: - logging.warn("TCP data %s decrypt %s" % (binascii.hexlify(ogn_data), binascii.hexlify(before_parse_data))) - raise Exception('can not parse header') + data = self._handel_protocol_error(self._client_address, ogn_data) data = pre_parse_header(data) if data is None: - logging.warn("TCP data %s decrypt %s" % (binascii.hexlify(ogn_data), binascii.hexlify(before_parse_data))) - raise Exception('can not parse header') + data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) if header_result is None: - logging.warn("TCP data %s decrypt %s" % (binascii.hexlify(ogn_data), binascii.hexlify(before_parse_data))) - raise Exception('can not parse header') + data = self._handel_protocol_error(self._client_address, ogn_data) + header_result = parse_header(data) connecttype, remote_addr, remote_port, header_length = header_result logging.info('%s connecting %s:%d from %s:%d' % ((connecttype == 0) and 'TCP' or 'UDP', @@ -505,7 +529,8 @@ def _on_local_read(self): ogn_data = data self._update_activity(len(data)) if not is_local: - data = self._encryptor.decrypt(data) + if self._encrypt_correct: + data = self._encryptor.decrypt(data) if not data: return self._server.server_transfer_ul += len(data) @@ -557,7 +582,8 @@ def _on_remote_read(self, is_remote_sock): if self._is_local: data = self._encryptor.decrypt(data) else: - data = self._encryptor.encrypt(data) + if self._encrypt_correct: + data = self._encryptor.encrypt(data) try: self._write_to_sock(data, self._local_sock) except Exception as e: From 5a5b8975786f852e34d01da8c4caf7ed302246ac Mon Sep 17 00:00:00 2001 From: falseen Date: Thu, 10 Sep 2015 02:00:43 +0800 Subject: [PATCH 036/487] =?UTF-8?q?=E5=85=BC=E5=AE=B9py3=EF=BC=8C=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E8=AF=A6=E7=BB=86=E9=94=99=E8=AF=AF=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 兼容py3，显示详细错误。 --- shadowsocks/crypto/util.py | 4 ++-- shadowsocks/tcprelay.py | 4 ++-- shadowsocks/udprelay.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shadowsocks/crypto/util.py b/shadowsocks/crypto/util.py index e579455e..a16baba5 100644 --- a/shadowsocks/crypto/util.py +++ b/shadowsocks/crypto/util.py @@ -87,8 +87,8 @@ def find_library(possible_lib_names, search_symbol, library_name): else: logging.warn('can\'t find symbol %s in %s', search_symbol, path) - except Exception: - pass + except ValueError as e: + print(e) return None diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index cecd0176..87710c13 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -707,8 +707,8 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._closed = False self._eventloop = None self._fd_to_handlers = {} - self.server_transfer_ul = 0L - self.server_transfer_dl = 0L + self.server_transfer_ul = 0 + self.server_transfer_dl = 0 self._timeout = config['timeout'] self._timeouts = [] # a list for all the handlers diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 3ea9b6d8..315a0b07 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -873,8 +873,8 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._dns_cache = lru_cache.LRUCache(timeout=300) self._eventloop = None self._closed = False - self.server_transfer_ul = 0L - self.server_transfer_dl = 0L + self.server_transfer_ul = 0 + self.server_transfer_dl = 0 self._sockets = set() self._fd_to_handlers = {} From b6961974ba39a912cd685c11264e9a1611a73c47 Mon Sep 17 00:00:00 2001 From: falseen Date: Fri, 11 Sep 2015 20:23:27 +0800 Subject: [PATCH 037/487] =?UTF-8?q?=E5=85=BC=E5=AE=B9py3=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E8=AF=AD=E6=B3=95=E9=94=99=E8=AF=AF=EF=BC=8C=E5=8A=A0?= =?UTF-8?q?=E8=BD=BDlibcrypto=E6=A8=A1=E5=9D=97=E5=A4=B1=E8=B4=A5=E6=97=B6?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E8=AF=A6=E7=BB=86=E9=94=99=E8=AF=AF=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正语法错误：` except Exception, e: ` 是旧版本的语法，py2.6之后的版本已不再使用此语法。 加载libcrypto模块失败时显示详细错误。 --- db_transfer.py | 6 +++--- server_pool.py | 14 +++++++------- shadowsocks/crypto/util.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index a62c30f8..ae65939c 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -79,7 +79,7 @@ def pull_db_all_user(): try: import switchrule keys = switchrule.getKeys() - except Exception, e: + except Exception as e: keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') @@ -102,13 +102,13 @@ def del_server_out_of_bound_safe(last_rows, rows): #需要动态载入switchrule，以便实时修改规则 try: import switchrule - except Exception, e: + except Exception as e: logging.error('load switchrule.py fail') cur_servers = {} for row in rows: try: allow = switchrule.isTurnOn(row) and row['enable'] == 1 and row['u'] + row['d'] < row['transfer_enable'] - except Exception, e: + except Exception as e: allow = False port = row['port'] diff --git a/server_pool.py b/server_pool.py index ee0e86b1..90101bd5 100644 --- a/server_pool.py +++ b/server_pool.py @@ -120,7 +120,7 @@ def new_server(self, port, password): if a_config['server_ipv6'] == "::": ipv6_ok = True - except Exception, e: + except Exception as e: logging.warn("IPV6 %s " % (e,)) if 'server' in self.config: @@ -142,7 +142,7 @@ def new_server(self, port, password): udp_server.add_to_loop(self.loop) self.udp_servers_pool.update({port: udp_server}) - except Exception, e: + except Exception as e: if not ipv6_ok: logging.warn("IPV4 %s " % (e,)) @@ -155,7 +155,7 @@ def del_server(self, port): udpsock = socket(AF_INET, SOCK_DGRAM) udpsock.sendto('%s:%s:0:0' % (Config.MANAGE_PASS, port), (Config.MANAGE_BIND_IP, Config.MANAGE_PORT)) udpsock.close() - except Exception, e: + except Exception as e: logging.warn(e) return True @@ -169,12 +169,12 @@ def cb_del_server(self, port): try: self.tcp_servers_pool[port].close(False) del self.tcp_servers_pool[port] - except Exception, e: + except Exception as e: logging.warn(e) try: self.udp_servers_pool[port].close(False) del self.udp_servers_pool[port] - except Exception, e: + except Exception as e: logging.warn(e) if 'server_ipv6' in self.config: @@ -185,12 +185,12 @@ def cb_del_server(self, port): try: self.tcp_ipv6_servers_pool[port].close(False) del self.tcp_ipv6_servers_pool[port] - except Exception, e: + except Exception as e: logging.warn(e) try: self.udp_ipv6_servers_pool[port].close(False) del self.udp_ipv6_servers_pool[port] - except Exception, e: + except Exception as e: logging.warn(e) return True diff --git a/shadowsocks/crypto/util.py b/shadowsocks/crypto/util.py index a16baba5..4deb6371 100644 --- a/shadowsocks/crypto/util.py +++ b/shadowsocks/crypto/util.py @@ -88,7 +88,7 @@ def find_library(possible_lib_names, search_symbol, library_name): logging.warn('can\'t find symbol %s in %s', search_symbol, path) except ValueError as e: - print(e) + logging.error(e) return None From a723c2f05edc3e3c2f205ef53f3b677535256219 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 11 Sep 2015 21:17:48 +0800 Subject: [PATCH 038/487] add obfs plugin --- config.json | 1 + shadowsocks/obfs.py | 61 ++++++++++++++++++ shadowsocks/obfsplugin/__init__.py | 18 ++++++ shadowsocks/obfsplugin/http_simple.py | 91 +++++++++++++++++++++++++++ shadowsocks/obfsplugin/plain.py | 41 ++++++++++++ shadowsocks/tcprelay.py | 18 ++++-- 6 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 shadowsocks/obfs.py create mode 100644 shadowsocks/obfsplugin/__init__.py create mode 100644 shadowsocks/obfsplugin/http_simple.py create mode 100644 shadowsocks/obfsplugin/plain.py diff --git a/config.json b/config.json index 57e9cc18..11b34c23 100644 --- a/config.json +++ b/config.json @@ -7,6 +7,7 @@ "password":"m", "timeout":300, "method":"aes-256-cfb", + "obfs":"http_simple", "fast_open": false, "workers": 1 } diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py new file mode 100644 index 00000000..c54eaa80 --- /dev/null +++ b/shadowsocks/obfs.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# +# Copyright 2015-2015 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import hashlib +import logging + +from shadowsocks import common +from shadowsocks.obfsplugin import plain, http_simple + + +method_supported = {} +method_supported.update(plain.obfs) +method_supported.update(http_simple.obfs) + +class Obfs(object): + def __init__(self, method): + self.method = method + self._method_info = self.get_method_info(method) + if self._method_info: + self.obfs = self.get_obfs(method) + else: + logging.error('method %s not supported' % method) + sys.exit(1) + + def get_method_info(self, method): + method = method.lower() + m = method_supported.get(method) + return m + + def get_obfs(self, method): + m = self._method_info + return m[0](method) + + def encode(self, buf): + #if len(buf) == 0: + # return buf + return self.obfs.encode(buf) + + def decode(self, buf): + #if len(buf) == 0: + # return (buf, True, False) + return self.obfs.decode(buf) + diff --git a/shadowsocks/obfsplugin/__init__.py b/shadowsocks/obfsplugin/__init__.py new file mode 100644 index 00000000..401c7b72 --- /dev/null +++ b/shadowsocks/obfsplugin/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# +# Copyright 2015 clowwindy +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py new file mode 100644 index 00000000..36e90288 --- /dev/null +++ b/shadowsocks/obfsplugin/http_simple.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# +# Copyright 2015-2015 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import hashlib +import logging +import binascii +import datetime + +def create_obfs(method): + return http_simple(method) + +obfs = { + 'http_simple': (create_obfs,), +} + +class http_simple(object): + def __init__(self, method): + self.method = method + self.has_sent_header = False + self.has_recv_header = False + self.host = "" + self.port = 0 + self.recv_buffer = "" + + def encode(self, buf): + if self.has_sent_header: + return buf + else: + header = "HTTP/1.1 200 OK\r\nServer: openresty\r\nDate: " + header += datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT') + header += '''\r\nContent-Type: text/plain; charset=utf-8\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\nKeep-Alive: timeout=20\r\nVary: Accept-Encoding\r\nContent-Encoding: gzip\r\n\r\n''' + self.has_sent_header = True + return header + buf + + def decode(self, buf): + if self.has_recv_header: + return (buf, True, False) + else: + buf = self.recv_buffer + buf + if len(buf) > 10: + if buf[:5] == "GET /" or buf[:6] == "POST /": + pass + else: #not http header, run on original protocol + self.has_sent_header = True + self.has_recv_header = True + self.recv_buffer = None + return (buf, True, False) + else: + self.recv_buffer = buf + return ("", True, False) + + datas = buf.split('\r\n\r\n', 1) + if datas and len(datas) > 1 and len(datas[1]) >= 7: + lines = buf.split('\r\n') + if lines and len(lines) > 4: + hex_items = lines[0].split('%') + if hex_items and len(hex_items) > 1: + ret_buf = "" + for index in xrange(1, len(hex_items)): + if len(hex_items[index]) != 2: + ret_buf += binascii.unhexlify(hex_items[index][:2]) + break + ret_buf += binascii.unhexlify(hex_items[index]) + ret_buf += datas[1] + self.has_recv_header = True + return (ret_buf, True, False) + else: + self.recv_buffer = buf + return ("", True, False) + self.has_sent_header = True + self.has_recv_header = True + return (buf, True, False) + diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py new file mode 100644 index 00000000..90b5ea0f --- /dev/null +++ b/shadowsocks/obfsplugin/plain.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# +# Copyright 2015-2015 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import hashlib +import logging + +def create_obfs(method): + return plain(method) + +obfs = { + 'plain': (create_obfs,), +} + +class plain(object): + def __init__(self, method): + self.method = method + + def encode(self, buf): + return buf + + def decode(self, buf): + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (buf, True, False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index cecd0176..3b073dc5 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -27,7 +27,7 @@ import traceback import random -from shadowsocks import encrypt, eventloop, shell, common +from shadowsocks import encrypt, obfs, eventloop, shell, common from shadowsocks.common import pre_parse_header, parse_header # set it 'False' to use both new protocol and the original shadowsocks protocal @@ -115,6 +115,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._encryptor = encrypt.Encryptor(config['password'], config['method']) self._encrypt_correct = True + self._obfs = obfs.Obfs(config.get('obfs', 'plain')) self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] @@ -197,7 +198,7 @@ def _write_to_sock(self, data, sock): # write data to sock # if only some of the data are written, put remaining in the buffer # and update the stream to wait for writing - if not data or not sock: + if not sock: return False #logging.debug("_write_to_sock %s %s %s" % (self._remote_sock, sock, self._remote_udp)) uncomplete = False @@ -249,6 +250,9 @@ def _write_to_sock(self, data, sock): return True else: try: + if sock == self._local_sock and self._encrypt_correct: + obfs_encode = self._obfs.encode(data) + data = obfs_encode l = len(data) s = sock.send(data) if s < l: @@ -298,13 +302,13 @@ def _get_redirect_host(self, client_address, ogn_data): return host_list[((hash_code & 0xffffffff) + addr + 3) % len(host_list)] def _handel_protocol_error(self, client_address, ogn_data): + #raise Exception('can not parse header') logging.warn("Protocol ERROR, TCP ogn data %s" % (binascii.hexlify(ogn_data), )) self._encrypt_correct = False #create redirect or disconnect by hash code host, port = self._get_redirect_host(client_address, ogn_data) data = "\x03" + chr(len(host)) + host + struct.pack('>H', port) logging.warn("TCP data redir %s:%d %s" % (host, port, binascii.hexlify(data))) - #raise Exception('can not parse header') return data + ogn_data def _handle_stage_connecting(self, data): @@ -530,7 +534,13 @@ def _on_local_read(self): self._update_activity(len(data)) if not is_local: if self._encrypt_correct: - data = self._encryptor.decrypt(data) + obfs_decode = self._obfs.decode(data) + if obfs_decode[2]: + self._write_to_sock("", self._local_sock) + if obfs_decode[1]: + data = self._encryptor.decrypt(obfs_decode[0]) + else: + data = obfs_decode[0] if not data: return self._server.server_transfer_ul += len(data) From 2e3114f40ff7b1c1460b6f935f6eafe1bde5b6b2 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Mon, 21 Sep 2015 14:41:42 +0800 Subject: [PATCH 039/487] fix 'ord' bug replace 'xrange' support new protocol when run as a client --- shadowsocks/obfs.py | 20 +++++++----- shadowsocks/obfsplugin/http_simple.py | 12 +++++-- shadowsocks/obfsplugin/plain.py | 11 +++++-- shadowsocks/tcprelay.py | 47 +++++++++++++++++++-------- shadowsocks/udprelay.py | 28 +++++++++------- 5 files changed, 79 insertions(+), 39 deletions(-) diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index c54eaa80..9d6d58ed 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -49,13 +49,15 @@ def get_obfs(self, method): m = self._method_info return m[0](method) - def encode(self, buf): - #if len(buf) == 0: - # return buf - return self.obfs.encode(buf) - - def decode(self, buf): - #if len(buf) == 0: - # return (buf, True, False) - return self.obfs.decode(buf) + def client_encode(self, buf): + return self.obfs.client_encode(buf) + + def client_decode(self, buf): + return self.obfs.client_decode(buf) + + def server_encode(self, buf): + return self.obfs.server_encode(buf) + + def server_decode(self, buf): + return self.obfs.server_decode(buf) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 36e90288..592811cd 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -40,7 +40,15 @@ def __init__(self, method): self.port = 0 self.recv_buffer = "" - def encode(self, buf): + def client_encode(self, buf): + # TODO + return buf + + def client_decode(self, buf): + # TODO + return (buf, False) + + def server_encode(self, buf): if self.has_sent_header: return buf else: @@ -50,7 +58,7 @@ def encode(self, buf): self.has_sent_header = True return header + buf - def decode(self, buf): + def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) else: diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index 90b5ea0f..5834f896 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -33,9 +33,16 @@ class plain(object): def __init__(self, method): self.method = method - def encode(self, buf): + def client_encode(self, buf): return buf - def decode(self, buf): + def client_decode(self, buf): + # (buffer_to_recv, is_need_to_encode_and_send_back) + return (buf, False) + + def server_encode(self, buf): + return buf + + def server_decode(self, buf): # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (buf, True, False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e304fe5e..04382499 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -33,6 +33,8 @@ # set it 'False' to use both new protocol and the original shadowsocks protocal # set it 'True' to use new protocol ONLY, to avoid GFW detecting FORCE_NEW_PROTOCOL = False +# set it 'True' if run as a local client and connect to a server which support new protocol +CLIENT_NEW_PROTOCOL = False # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 @@ -250,9 +252,12 @@ def _write_to_sock(self, data, sock): return True else: try: - if sock == self._local_sock and self._encrypt_correct: - obfs_encode = self._obfs.encode(data) - data = obfs_encode + if self._is_local: + pass + else: + if sock == self._local_sock and self._encrypt_correct: + obfs_encode = self._obfs.server_encode(data) + data = obfs_encode l = len(data) s = sock.send(data) if s < l: @@ -303,7 +308,7 @@ def _get_redirect_host(self, client_address, ogn_data): def _handel_protocol_error(self, client_address, ogn_data): #raise Exception('can not parse header') - logging.warn("Protocol ERROR, TCP ogn data %s" % (binascii.hexlify(ogn_data), )) + logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1])) self._encrypt_correct = False #create redirect or disconnect by hash code host, port = self._get_redirect_host(client_address, ogn_data) @@ -314,6 +319,7 @@ def _handel_protocol_error(self, client_address, ogn_data): def _handle_stage_connecting(self, data): if self._is_local: data = self._encryptor.encrypt(data) + data = self._obfs.client_encode(data) self._data_to_write_to_remote.append(data) if self._is_local and not self._fastopen_connected and \ self._config['fast_open']: @@ -377,15 +383,18 @@ def _handle_stage_addr(self, ogn_data, data): return before_parse_data = data - if FORCE_NEW_PROTOCOL and ord(data[0]) != 0x88: - data = self._handel_protocol_error(self._client_address, ogn_data) - data = pre_parse_header(data) - if data is None: - data = self._handel_protocol_error(self._client_address, ogn_data) - header_result = parse_header(data) - if header_result is None: - data = self._handel_protocol_error(self._client_address, ogn_data) + if self._is_local: header_result = parse_header(data) + else: + if FORCE_NEW_PROTOCOL and common.ord(data[0]) != 0x88: + data = self._handel_protocol_error(self._client_address, ogn_data) + data = pre_parse_header(data) + if data is None: + data = self._handel_protocol_error(self._client_address, ogn_data) + header_result = parse_header(data) + if header_result is None: + data = self._handel_protocol_error(self._client_address, ogn_data) + header_result = parse_header(data) connecttype, remote_addr, remote_port, header_length = header_result logging.info('%s connecting %s:%d from %s:%d' % ((connecttype == 0) and 'TCP' or 'UDP', @@ -401,6 +410,12 @@ def _handle_stage_addr(self, ogn_data, data): self._write_to_sock((b'\x05\x00\x00\x01' b'\x00\x00\x00\x00\x10\x10'), self._local_sock) + if CLIENT_NEW_PROTOCOL: + rnd_len = random.randint(1, 32) + total_len = 7 + rnd_len + len(data) + data = b'\x88' + struct.pack('>H', total_len) + chr(rnd_len) + (b' ' * (rnd_len - 1)) + data + crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff + data += struct.pack(' self.end_id: - for eid in xrange(self.end_id, end_id): + eid = self.end_id + while eid < pack_id: self.miss_queue.add(eid) + eid += 1 self.end_id = end_id def get_begin_id(self): @@ -302,7 +306,7 @@ def __init__(self, server, reqid_to_handlers, fd_to_handlers, loop, #loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) self.last_activity = 0 self._update_activity() - self._random_mtu_size = [random.randint(POST_MTU_MIN, POST_MTU_MAX) for i in xrange(1024)] + self._random_mtu_size = [random.randint(POST_MTU_MIN, POST_MTU_MAX) for i in range(1024)] self._random_mtu_index = 0 self._rand_data = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" * 4 @@ -474,7 +478,7 @@ def _handle_dns_resolved(self, result, error): addr = self.get_local_address() - for i in xrange(2): + for i in range(2): rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) self._write_to_sock(rsp_data, self._local_sock, addr) @@ -671,7 +675,7 @@ def handle_client(self, addr, cmd, request_id, data): if self._stage == STAGE_RSP_ID: if cmd == CMD_CONNECT: - for i in xrange(2): + for i in range(2): rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, RSP_STATE_CONNECTED) self._write_to_sock(rsp_data, self._local_sock, addr) elif cmd == CMD_CONNECT_REMOTE: @@ -695,7 +699,7 @@ def handle_client(self, addr, cmd, request_id, data): if cmd == CMD_CONNECT_REMOTE: local_id = data[0:4] if self._local_id == local_id: - for i in xrange(2): + for i in range(2): rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) self._write_to_sock(rsp_data, self._local_sock, addr) else: @@ -927,7 +931,7 @@ def _close_client(self, client): def _pre_parse_udp_header(self, data): if data is None: return - datatype = ord(data[0]) + datatype = common.ord(data[0]) if datatype == 0x8: if len(data) >= 8: crc = binascii.crc32(data) & 0xffffffff @@ -935,17 +939,17 @@ def _pre_parse_udp_header(self, data): logging.warn('uncorrect CRC32, maybe wrong password or ' 'encryption method') return None - cmd = ord(data[1]) + cmd = common.ord(data[1]) request_id = struct.unpack('>H', data[2:4])[0] data = data[4:-4] return (cmd, request_id, data) - elif len(data) >= 6 and ord(data[1]) == 0x0: + elif len(data) >= 6 and common.ord(data[1]) == 0x0: crc = binascii.crc32(data) & 0xffffffff if crc != 0xffffffff: logging.warn('uncorrect CRC32, maybe wrong password or ' 'encryption method') return None - cmd = ord(data[1]) + cmd = common.ord(data[1]) data = data[2:-4] return (cmd, 0, data) else: @@ -994,12 +998,12 @@ def _handle_server(self): try: if data[0] == 0: if len(data[2]) >= 4: - for i in xrange(64): + for i in range(64): req_id = random.randint(1, 65535) if req_id not in self._reqid_to_hd: break if req_id in self._reqid_to_hd: - for i in xrange(64): + for i in range(64): req_id = random.randint(1, 65535) if type(self._reqid_to_hd[req_id]) is tuple: break From 71338a738539dc338a83e14ee1567baa313459fe Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 25 Sep 2015 11:28:09 +0800 Subject: [PATCH 040/487] fix on python3 http2 obfs --- shadowsocks/manager.py | 2 +- shadowsocks/obfsplugin/http_simple.py | 127 +++++++++++++++++++++----- shadowsocks/shell.py | 1 + shadowsocks/tcprelay.py | 16 ++-- shadowsocks/udprelay.py | 38 ++++---- 5 files changed, 133 insertions(+), 51 deletions(-) diff --git a/shadowsocks/manager.py b/shadowsocks/manager.py index e8009b48..d5d9d12c 100644 --- a/shadowsocks/manager.py +++ b/shadowsocks/manager.py @@ -44,7 +44,7 @@ def __init__(self, config): self._statistics = collections.defaultdict(int) self._control_client_addr = None try: - manager_address = config['manager_address'] + manager_address = common.to_str(config['manager_address']) if ':' in manager_address: addr = manager_address.rsplit(':', 1) addr = addr[0], int(addr[1]) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 592811cd..90f184fb 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -22,23 +22,35 @@ import hashlib import logging import binascii +import base64 import datetime +from shadowsocks.common import to_bytes, to_str -def create_obfs(method): +def create_http_obfs(method): return http_simple(method) +def create_http2_obfs(method): + return http2_simple(method) + obfs = { - 'http_simple': (create_obfs,), + 'http_simple': (create_http_obfs,), + 'http2_simple': (create_http2_obfs,), } +def match_begin(str1, str2): + if len(str1) >= len(str2): + if str1[:len(str2)] == str2: + return True + return False + class http_simple(object): def __init__(self, method): self.method = method self.has_sent_header = False self.has_recv_header = False - self.host = "" + self.host = None self.port = 0 - self.recv_buffer = "" + self.recv_buffer = b'' def client_encode(self, buf): # TODO @@ -52,19 +64,33 @@ def server_encode(self, buf): if self.has_sent_header: return buf else: - header = "HTTP/1.1 200 OK\r\nServer: openresty\r\nDate: " - header += datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT') - header += '''\r\nContent-Type: text/plain; charset=utf-8\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\nKeep-Alive: timeout=20\r\nVary: Accept-Encoding\r\nContent-Encoding: gzip\r\n\r\n''' + header = b'HTTP/1.1 200 OK\r\nServer: openresty\r\nDate: ' + header += to_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT')) + header += b'\r\nContent-Type: text/plain; charset=utf-8\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\nKeep-Alive: timeout=20\r\nVary: Accept-Encoding\r\nContent-Encoding: gzip\r\n\r\n' self.has_sent_header = True return header + buf + def get_data_from_http_header(self, buf): + ret_buf = b'' + lines = buf.split(b'\r\n') + if lines and len(lines) > 4: + hex_items = lines[0].split(b'%') + if hex_items and len(hex_items) > 1: + for index in range(1, len(hex_items)): + if len(hex_items[index]) != 2: + ret_buf += binascii.unhexlify(hex_items[index][:2]) + break + ret_buf += binascii.unhexlify(hex_items[index]) + return ret_buf + return b'' + def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) else: buf = self.recv_buffer + buf if len(buf) > 10: - if buf[:5] == "GET /" or buf[:6] == "POST /": + if match_begin(buf, b'GET /') or match_begin(buf, b'POST /'): pass else: #not http header, run on original protocol self.has_sent_header = True @@ -73,26 +99,81 @@ def server_decode(self, buf): return (buf, True, False) else: self.recv_buffer = buf - return ("", True, False) - - datas = buf.split('\r\n\r\n', 1) - if datas and len(datas) > 1 and len(datas[1]) >= 7: - lines = buf.split('\r\n') - if lines and len(lines) > 4: - hex_items = lines[0].split('%') - if hex_items and len(hex_items) > 1: - ret_buf = "" - for index in xrange(1, len(hex_items)): - if len(hex_items[index]) != 2: - ret_buf += binascii.unhexlify(hex_items[index][:2]) - break - ret_buf += binascii.unhexlify(hex_items[index]) + return (b'', True, False) + + datas = buf.split(b'\r\n\r\n', 1) + ret_buf = b'' + if datas and len(datas) > 1: + ret_buf = self.get_data_from_http_header(buf) + ret_buf += datas[1] + if len(ret_buf) >= 15: + self.has_recv_header = True + return (ret_buf, True, False) + self.recv_buffer = buf + return (b'', True, False) + else: + self.recv_buffer = buf + return (b'', True, False) + self.has_sent_header = True + self.has_recv_header = True + return (buf, True, False) + +class http2_simple(object): + def __init__(self, method): + self.method = method + self.has_sent_header = False + self.has_recv_header = False + self.host = None + self.port = 0 + self.recv_buffer = b'' + + def client_encode(self, buf): + # TODO + return buf + + def client_decode(self, buf): + # TODO + return (buf, False) + + def server_encode(self, buf): + if self.has_sent_header: + return buf + else: + header = b'HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: h2c\r\n\r\n' + self.has_sent_header = True + return header + buf + + def server_decode(self, buf): + if self.has_recv_header: + return (buf, True, False) + else: + buf = self.recv_buffer + buf + if len(buf) > 10: + if match_begin(buf, b'GET /'): + pass + else: #not http header, run on original protocol + self.has_sent_header = True + self.has_recv_header = True + self.recv_buffer = None + return (buf, True, False) + else: + self.recv_buffer = buf + return (b'', True, False) + + datas = buf.split(b'\r\n\r\n', 1) + if datas and len(datas) > 1 and len(datas[0]) >= 4: + lines = buf.split(b'\r\n') + if lines and len(lines) >= 4: + if match_begin(lines[4], b'HTTP2-Settings: '): + ret_buf = base64.urlsafe_b64decode(lines[4][16:]) ret_buf += datas[1] self.has_recv_header = True return (ret_buf, True, False) + self.recv_buffer = buf + return (b'', True, False) else: self.recv_buffer = buf - return ("", True, False) + return (b'', True, False) self.has_sent_header = True self.has_recv_header = True return (buf, True, False) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index c91fc222..e5f5bb52 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -216,6 +216,7 @@ def get_config(is_local): config['password'] = to_bytes(config.get('password', b'')) config['method'] = to_str(config.get('method', 'aes-256-cfb')) + config['obfs'] = to_str(config.get('obfs', 'plain')) config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) config['fast_open'] = config.get('fast_open', False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 04382499..1f5647c3 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -117,11 +117,11 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._encryptor = encrypt.Encryptor(config['password'], config['method']) self._encrypt_correct = True - self._obfs = obfs.Obfs(config.get('obfs', 'plain')) + self._obfs = obfs.Obfs(config['obfs']) self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] - self._udp_data_send_buffer = '' + self._udp_data_send_buffer = b'' self._upstream_status = WAIT_STATUS_READING self._downstream_status = WAIT_STATUS_INIT self._client_address = local_sock.getpeername()[:2] @@ -293,7 +293,7 @@ def _write_to_sock(self, data, sock): def _get_redirect_host(self, client_address, ogn_data): # test - host_list = [("www.bing.com", 80), ("www.microsoft.com", 80), ("www.baidu.com", 443), ("www.qq.com", 80), ("www.csdn.net", 80), ("1.2.3.4", 1000)] + host_list = [(b"www.bing.com", 80), (b"www.microsoft.com", 80), (b"www.baidu.com", 443), (b"www.qq.com", 80), (b"www.csdn.net", 80), (b"1.2.3.4", 1000)] hash_code = binascii.crc32(ogn_data) addrs = socket.getaddrinfo(client_address[0], client_address[1], 0, socket.SOCK_STREAM, socket.SOL_TCP) af, socktype, proto, canonname, sa = addrs[0] @@ -312,7 +312,7 @@ def _handel_protocol_error(self, client_address, ogn_data): self._encrypt_correct = False #create redirect or disconnect by hash code host, port = self._get_redirect_host(client_address, ogn_data) - data = "\x03" + chr(len(host)) + host + struct.pack('>H', port) + data = b"\x03" + common.chr(len(host)) + host + struct.pack('>H', port) logging.warn("TCP data redir %s:%d %s" % (host, port, binascii.hexlify(data))) return data + ogn_data @@ -551,7 +551,7 @@ def _on_local_read(self): if self._encrypt_correct: obfs_decode = self._obfs.server_decode(data) if obfs_decode[2]: - self._write_to_sock("", self._local_sock) + self._write_to_sock(b'', self._local_sock) if obfs_decode[1]: data = self._encryptor.decrypt(obfs_decode[0]) else: @@ -588,10 +588,10 @@ def _on_remote_read(self, is_remote_sock): port = struct.pack('>H', addr[1]) try: ip = socket.inet_aton(addr[0]) - data = '\x00\x01' + ip + port + data + data = b'\x00\x01' + ip + port + data except Exception as e: ip = socket.inet_pton(socket.AF_INET6, addr[0]) - data = '\x00\x04' + ip + port + data + data = b'\x00\x04' + ip + port + data data = struct.pack('>H', len(data) + 2) + data #logging.info('UDP over TCP recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) else: @@ -608,7 +608,7 @@ def _on_remote_read(self, is_remote_sock): if self._is_local: obfs_decode = self._obfs.client_decode(data) if obfs_decode[1]: - self._write_to_sock("", self._remote_sock) + self._write_to_sock(b'', self._remote_sock) data = self._encryptor.decrypt(obfs_decode[0]) else: if self._encrypt_correct: diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 5ebf7316..d7626c9c 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -115,15 +115,15 @@ CMD_SYN_STATUS_64 = 7 CMD_DISCONNECT = 8 -CMD_VER_STR = "\x08" +CMD_VER_STR = b"\x08" -RSP_STATE_EMPTY = "" -RSP_STATE_REJECT = "\x00" -RSP_STATE_CONNECTED = "\x01" -RSP_STATE_CONNECTEDREMOTE = "\x02" -RSP_STATE_ERROR = "\x03" -RSP_STATE_DISCONNECT = "\x04" -RSP_STATE_REDIRECT = "\x05" +RSP_STATE_EMPTY = b"" +RSP_STATE_REJECT = b"\x00" +RSP_STATE_CONNECTED = b"\x01" +RSP_STATE_CONNECTEDREMOTE = b"\x02" +RSP_STATE_ERROR = b"\x03" +RSP_STATE_DISCONNECT = b"\x04" +RSP_STATE_REDIRECT = b"\x05" class UDPLocalAddress(object): def __init__(self, addr): @@ -309,7 +309,7 @@ def __init__(self, server, reqid_to_handlers, fd_to_handlers, loop, self._random_mtu_size = [random.randint(POST_MTU_MIN, POST_MTU_MAX) for i in range(1024)] self._random_mtu_index = 0 - self._rand_data = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" * 4 + self._rand_data = b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" * 4 def __hash__(self): # default __hash__ is id / 16 @@ -589,29 +589,29 @@ def _on_remote_error(self): def _pack_rsp_data(self, cmd, data): reqid_str = struct.pack(">H", self._request_id) - return ''.join([CMD_VER_STR, chr(cmd), reqid_str, data, self._rand_data[:random.randint(0, len(self._rand_data))], reqid_str]) + return b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, data, self._rand_data[:random.randint(0, len(self._rand_data))], reqid_str]) def _pack_rnd_data(self, data): length = random.randint(0, len(self._rand_data)) if length == 0: return data elif length == 1: - return "\x81" + data + return b"\x81" + data elif length < 256: - return "\x80" + chr(length) + self._rand_data[:length - 2] + data + return b"\x80" + common.chr(length) + self._rand_data[:length - 2] + data else: - return "\x82" + struct.pack(">H", length) + self._rand_data[:length - 3] + data + return b"\x82" + struct.pack(">H", length) + self._rand_data[:length - 3] + data def _pack_post_data(self, cmd, pack_id, data): reqid_str = struct.pack(">H", self._request_id) recv_id = self._recvqueue.get_begin_id() - rsp_data = ''.join([CMD_VER_STR, chr(cmd), reqid_str, struct.pack(">I", recv_id), struct.pack(">I", pack_id), data, reqid_str]) + rsp_data = b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, struct.pack(">I", recv_id), struct.pack(">I", pack_id), data, reqid_str]) return rsp_data def _pack_post_data_64(self, cmd, send_id, pack_id, data): reqid_str = struct.pack(">H", self._request_id) recv_id = self._recvqueue.get_begin_id() - rsp_data = ''.join([CMD_VER_STR, chr(cmd), reqid_str, struct.pack(">Q", recv_id), struct.pack(">Q", pack_id), data, reqid_str]) + rsp_data = b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, struct.pack(">Q", recv_id), struct.pack(">Q", pack_id), data, reqid_str]) return rsp_data def sweep_timeout(self): @@ -619,7 +619,7 @@ def sweep_timeout(self): if self._stage == STAGE_STREAM: pack_id, missing = self._recvqueue.get_missing_id(0) logging.info("sweep_timeout %s %s" % (pack_id, missing)) - data = '' + data = b'' for pid in missing: data += struct.pack(">H", pid) rsp_data = self._pack_post_data(CMD_SYN_STATUS, pack_id, data) @@ -642,7 +642,7 @@ def handle_stream_sync_status(self, addr, cmd, request_id, pack_id, max_send_id, # post CMD_SYN_STATUS send_id = self._sendingqueue.get_end_id() post_pack_id, missing = self._recvqueue.get_missing_id(0) - pack_ids_data = '' + pack_ids_data = b'' for pid in missing: pack_ids_data += struct.pack(">H", pid) @@ -959,9 +959,9 @@ def _pre_parse_udp_header(self, data): return data def _pack_rsp_data(self, cmd, request_id, data): - _rand_data = "123456789abcdefghijklmnopqrstuvwxyz" * 2 + _rand_data = b"123456789abcdefghijklmnopqrstuvwxyz" * 2 reqid_str = struct.pack(">H", request_id) - return ''.join([CMD_VER_STR, chr(cmd), reqid_str, data, _rand_data[:random.randint(0, len(_rand_data))], reqid_str]) + return b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, data, _rand_data[:random.randint(0, len(_rand_data))], reqid_str]) def _handle_server(self): server = self._server_socket From 3e5cbb955eff8c5d5ce419df74b09c7f6d766ef5 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 25 Sep 2015 21:29:11 +0800 Subject: [PATCH 041/487] add obfs: 'tls_simple' --- shadowsocks/obfsplugin/http_simple.py | 165 ++++++++++++++++---------- shadowsocks/tcprelay.py | 11 +- 2 files changed, 108 insertions(+), 68 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 90f184fb..4dff7633 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -32,9 +32,13 @@ def create_http_obfs(method): def create_http2_obfs(method): return http2_simple(method) +def create_tls_obfs(method): + return tls_simple(method) + obfs = { 'http_simple': (create_http_obfs,), 'http2_simple': (create_http2_obfs,), + 'tls_simple': (create_tls_obfs,), } def match_begin(str1, str2): @@ -63,12 +67,12 @@ def client_decode(self, buf): def server_encode(self, buf): if self.has_sent_header: return buf - else: - header = b'HTTP/1.1 200 OK\r\nServer: openresty\r\nDate: ' - header += to_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT')) - header += b'\r\nContent-Type: text/plain; charset=utf-8\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\nKeep-Alive: timeout=20\r\nVary: Accept-Encoding\r\nContent-Encoding: gzip\r\n\r\n' - self.has_sent_header = True - return header + buf + + header = b'HTTP/1.1 200 OK\r\nServer: openresty\r\nDate: ' + header += to_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT')) + header += b'\r\nContent-Type: text/plain; charset=utf-8\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\nKeep-Alive: timeout=20\r\nVary: Accept-Encoding\r\nContent-Encoding: gzip\r\n\r\n' + self.has_sent_header = True + return header + buf def get_data_from_http_header(self, buf): ret_buf = b'' @@ -87,36 +91,39 @@ def get_data_from_http_header(self, buf): def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) - else: - buf = self.recv_buffer + buf - if len(buf) > 10: - if match_begin(buf, b'GET /') or match_begin(buf, b'POST /'): - pass - else: #not http header, run on original protocol + + buf = self.recv_buffer + buf + if len(buf) > 10: + if match_begin(buf, b'GET /') or match_begin(buf, b'POST /'): + if len(buf) > 65536: self.has_sent_header = True self.has_recv_header = True self.recv_buffer = None return (buf, True, False) - else: - self.recv_buffer = buf - return (b'', True, False) - - datas = buf.split(b'\r\n\r\n', 1) - ret_buf = b'' - if datas and len(datas) > 1: - ret_buf = self.get_data_from_http_header(buf) - ret_buf += datas[1] - if len(ret_buf) >= 15: - self.has_recv_header = True - return (ret_buf, True, False) - self.recv_buffer = buf - return (b'', True, False) - else: - self.recv_buffer = buf - return (b'', True, False) - self.has_sent_header = True - self.has_recv_header = True - return (buf, True, False) + else: #not http header, run on original protocol + self.has_sent_header = True + self.has_recv_header = True + self.recv_buffer = None + return (buf, True, False) + else: + self.recv_buffer = buf + return (b'', True, False) + + datas = buf.split(b'\r\n\r\n', 1) + if datas and len(datas) > 1: + ret_buf = self.get_data_from_http_header(buf) + ret_buf += datas[1] + if len(ret_buf) >= 15: + self.has_recv_header = True + return (ret_buf, True, False) + self.recv_buffer = buf + return (b'', True, False) + else: + self.recv_buffer = buf + return (b'', True, False) + self.has_sent_header = True + self.has_recv_header = True + return (buf, True, False) class http2_simple(object): def __init__(self, method): @@ -138,43 +145,73 @@ def client_decode(self, buf): def server_encode(self, buf): if self.has_sent_header: return buf - else: - header = b'HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: h2c\r\n\r\n' - self.has_sent_header = True - return header + buf + + header = b'HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: h2c\r\n\r\n' + self.has_sent_header = True + return header + buf def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) + + buf = self.recv_buffer + buf + if len(buf) > 10: + if match_begin(buf, b'GET /'): + pass + else: #not http header, run on original protocol + self.has_sent_header = True + self.has_recv_header = True + self.recv_buffer = None + return (buf, True, False) else: - buf = self.recv_buffer + buf - if len(buf) > 10: - if match_begin(buf, b'GET /'): - pass - else: #not http header, run on original protocol - self.has_sent_header = True + self.recv_buffer = buf + return (b'', True, False) + + datas = buf.split(b'\r\n\r\n', 1) + if datas and len(datas) > 1 and len(datas[0]) >= 4: + lines = buf.split(b'\r\n') + if lines and len(lines) >= 4: + if match_begin(lines[4], b'HTTP2-Settings: '): + ret_buf = base64.urlsafe_b64decode(lines[4][16:]) + ret_buf += datas[1] self.has_recv_header = True - self.recv_buffer = None - return (buf, True, False) - else: - self.recv_buffer = buf - return (b'', True, False) - - datas = buf.split(b'\r\n\r\n', 1) - if datas and len(datas) > 1 and len(datas[0]) >= 4: - lines = buf.split(b'\r\n') - if lines and len(lines) >= 4: - if match_begin(lines[4], b'HTTP2-Settings: '): - ret_buf = base64.urlsafe_b64decode(lines[4][16:]) - ret_buf += datas[1] - self.has_recv_header = True - return (ret_buf, True, False) - self.recv_buffer = buf - return (b'', True, False) - else: - self.recv_buffer = buf - return (b'', True, False) - self.has_sent_header = True - self.has_recv_header = True + return (ret_buf, True, False) + self.recv_buffer = buf + return (b'', True, False) + else: + self.recv_buffer = buf + return (b'', True, False) + self.has_sent_header = True + self.has_recv_header = True + return (buf, True, False) + +class tls_simple(object): + def __init__(self, method): + self.method = method + self.has_sent_header = False + self.has_recv_header = False + + def client_encode(self, buf): + return buf + + def client_decode(self, buf): + # (buffer_to_recv, is_need_to_encode_and_send_back) + return (buf, False) + + def server_encode(self, buf): + if self.has_sent_header: + return buf + self.has_sent_header = True + # TODO + #server_hello = b'' + return b'\x16\x03\x01' + + def server_decode(self, buf): + if self.has_recv_header: return (buf, True, False) + self.has_recv_header = True + if not match_begin(buf, b'\x16\x03\x01'): + return (buf, True, False) + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (b'', False, True) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 1f5647c3..7e2b3050 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -259,10 +259,13 @@ def _write_to_sock(self, data, sock): obfs_encode = self._obfs.server_encode(data) data = obfs_encode l = len(data) - s = sock.send(data) - if s < l: - data = data[s:] - uncomplete = True + if l > 0: + s = sock.send(data) + if s < l: + data = data[s:] + uncomplete = True + else: + return except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) if error_no in (errno.EAGAIN, errno.EINPROGRESS, From 6f8fc5b77f39e2567667bb9ff6340fd11166f55c Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 25 Sep 2015 22:17:34 +0800 Subject: [PATCH 042/487] remove thread module --- server.py | 12 ++++++++++-- server_pool.py | 12 ++++++++++-- shadowsocks/server.py | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/server.py b/server.py index de03e474..585380f9 100644 --- a/server.py +++ b/server.py @@ -1,7 +1,7 @@ #!/usr/bin/python import time import sys -import thread +import threading import os os.chdir(os.path.split(os.path.realpath(__file__))[0]) @@ -12,9 +12,17 @@ # thread.start_new_thread(DbTransfer.thread_db, ()) # Api.web_server() +class MainThread(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + + def run(self): + db_transfer.DbTransfer.thread_db() + if __name__ == '__main__': #server_pool.ServerPool.get_instance() #server_pool.ServerPool.get_instance().new_server(2333, '2333') - thread.start_new_thread(db_transfer.DbTransfer.thread_db, ()) + thread = MainThread() + thread.start() while True: time.sleep(99999) diff --git a/server_pool.py b/server_pool.py index 90101bd5..77d300d7 100644 --- a/server_pool.py +++ b/server_pool.py @@ -29,13 +29,20 @@ from shadowsocks import tcprelay from shadowsocks import udprelay from shadowsocks import asyncdns -import thread import threading import sys import asyncmgr import Config from socket import * +class MainThread(threading.Thread): + def __init__(self, params): + threading.Thread.__init__(self) + self.params = params + + def run(self): + ServerPool._loop(*self.params) + class ServerPool(object): instance = None @@ -53,7 +60,8 @@ def __init__(self): self.udp_ipv6_servers_pool = {} self.loop = eventloop.EventLoop() - thread.start_new_thread(ServerPool._loop, (self.loop, self.dns_resolver, self.mgr)) + thread = MainThread( (self.loop, self.dns_resolver, self.mgr) ) + thread.start() @staticmethod def get_instance(): diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 68a6716b..e61d6c29 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -74,7 +74,7 @@ def main(): (a_config['server'], int(port))) tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) - if a_config['server_ipv6'] == "::": + if a_config['server_ipv6'] == b"::": ipv6_ok = True except Exception as e: shell.print_exception(e) From 680c128c72ba2e0af97d46ead389e4f857f5448d Mon Sep 17 00:00:00 2001 From: falseen Date: Fri, 25 Sep 2015 23:17:01 +0800 Subject: [PATCH 043/487] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=86=E8=8A=82?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复：搜索完所有路径之后再判断是否抛出错误，而不是遇到错误就抛出。主要是考虑到系统中同时存在32位和64位dll的情况，原来的代码如果第一个找到的dll文件跟系统不符的话会直接报错。 --- shadowsocks/crypto/util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shadowsocks/crypto/util.py b/shadowsocks/crypto/util.py index 4deb6371..212df860 100644 --- a/shadowsocks/crypto/util.py +++ b/shadowsocks/crypto/util.py @@ -87,8 +87,9 @@ def find_library(possible_lib_names, search_symbol, library_name): else: logging.warn('can\'t find symbol %s in %s', search_symbol, path) - except ValueError as e: - logging.error(e) + except Exception: + if path == paths[-1]: + raise return None From f9e8af6358c2143273e330b7c1e0fb7a9e7e3f36 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 25 Sep 2015 23:25:18 +0800 Subject: [PATCH 044/487] add random_head obfs --- shadowsocks/obfsplugin/http_simple.py | 34 ++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 4dff7633..7b3cf28b 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -24,7 +24,8 @@ import binascii import base64 import datetime -from shadowsocks.common import to_bytes, to_str +from shadowsocks import common +from shadowsocks.common import to_bytes, to_str, ord def create_http_obfs(method): return http_simple(method) @@ -35,10 +36,14 @@ def create_http2_obfs(method): def create_tls_obfs(method): return tls_simple(method) +def create_random_head_obfs(method): + return random_head(method) + obfs = { 'http_simple': (create_http_obfs,), 'http2_simple': (create_http2_obfs,), 'tls_simple': (create_tls_obfs,), + 'random_head': (create_random_head_obfs,), } def match_begin(str1, str2): @@ -215,3 +220,30 @@ def server_decode(self, buf): return (buf, True, False) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) + +class random_head(object): + def __init__(self, method): + self.method = method + self.has_sent_header = False + self.has_recv_header = False + + def client_encode(self, buf): + return buf + + def client_decode(self, buf): + # (buffer_to_recv, is_need_to_encode_and_send_back) + return (buf, False) + + def server_encode(self, buf): + if self.has_sent_header: + return buf + self.has_sent_header = True + return os.urandom(common.ord(os.urandom(1)[0]) % 96) + + def server_decode(self, buf): + if self.has_recv_header: + return (buf, True, False) + + self.has_recv_header = True + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (b'', False, True) From e060fd5b4d9e7f0ab4921b3dc9212de83816cff4 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Sat, 26 Sep 2015 00:04:08 +0800 Subject: [PATCH 045/487] tiny bug --- shadowsocks/obfsplugin/http_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 7b3cf28b..0e3c211d 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -238,7 +238,7 @@ def server_encode(self, buf): if self.has_sent_header: return buf self.has_sent_header = True - return os.urandom(common.ord(os.urandom(1)[0]) % 96) + return os.urandom(common.ord(os.urandom(1)[0]) % 96 + 1) def server_decode(self, buf): if self.has_recv_header: From f9cdf2090fd38a5ba78aee366662552d639a11d2 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Sat, 26 Sep 2015 22:33:11 +0800 Subject: [PATCH 046/487] add crc verify --- shadowsocks/obfsplugin/http_simple.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 0e3c211d..367d7ccb 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -238,12 +238,15 @@ def server_encode(self, buf): if self.has_sent_header: return buf self.has_sent_header = True - return os.urandom(common.ord(os.urandom(1)[0]) % 96 + 1) + return os.urandom(common.ord(os.urandom(1)[0]) % 96 + 4) def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) + crc = binascii.crc32(buf) & 0xffffffff + if crc != 0xffffffff: + return (buf, True, False) self.has_recv_header = True # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) From 74972875ffb09de46e6dbd50a8677ffff48f31d0 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Mon, 28 Sep 2015 17:48:27 +0800 Subject: [PATCH 047/487] tiny fix --- shadowsocks/obfsplugin/http_simple.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 367d7ccb..6df2f2ef 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -217,6 +217,7 @@ def server_decode(self, buf): self.has_recv_header = True if not match_begin(buf, b'\x16\x03\x01'): + self.has_sent_header = True return (buf, True, False) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) @@ -244,9 +245,10 @@ def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) + self.has_recv_header = True crc = binascii.crc32(buf) & 0xffffffff if crc != 0xffffffff: + self.has_sent_header = True return (buf, True, False) - self.has_recv_header = True # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) From 96a9a25eaab10925a467be9d3b0a2df1c1c3bc95 Mon Sep 17 00:00:00 2001 From: falseen Date: Wed, 30 Sep 2015 01:01:19 +0800 Subject: [PATCH 048/487] =?UTF-8?q?=E5=A4=9A=E7=AB=AF=E5=8F=A3=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E4=B8=8B=E6=94=AF=E6=8C=81=E6=AF=8F=E4=B8=AA=E7=AB=AF?= =?UTF-8?q?=E5=8F=A3=E5=AE=9A=E4=B9=89=E4=B8=8D=E5=90=8C=E7=9A=84=E6=B7=B7?= =?UTF-8?q?=E6=B7=86=E5=8D=8F=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加：多端口模式下支持每个端口定义不同的混淆协议 --- shadowsocks/server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index e61d6c29..006a3723 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -59,16 +59,23 @@ def main(): dns_resolver = asyncdns.DNSResolver() port_password = config['port_password'] del config['port_password'] - for port, password in port_password.items(): + for port, password_obfs in port_password.items(): + if type(password_obfs) == list: + password = password_obfs[0] + obfs = password_obfs[1] + else: + password = password_obfs + obfs = config["obfs"] a_config = config.copy() ipv6_ok = False - logging.info("server start with password [%s] method [%s]" % (password, a_config['method'])) + logging.info("server start with password [%s] obfs [%s] method [%s]" % (password, obfs, a_config['method'])) if 'server_ipv6' in a_config: try: if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": a_config['server_ipv6'] = a_config['server_ipv6'][1:-1] a_config['server_port'] = int(port) a_config['password'] = password + a_config['obfs'] = obfs a_config['server'] = a_config['server_ipv6'] logging.info("starting server at %s:%d" % (a_config['server'], int(port))) @@ -83,6 +90,7 @@ def main(): a_config = config.copy() a_config['server_port'] = int(port) a_config['password'] = password + a_config['obfs'] = obfs logging.info("starting server at %s:%d" % (a_config['server'], int(port))) tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) From 463ce07426f3220e9dfe301f23a859273e5837ff Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Sun, 4 Oct 2015 19:56:58 +0800 Subject: [PATCH 049/487] add verify_simple module --- shadowsocks/obfs.py | 15 +++- shadowsocks/obfsplugin/http_simple.py | 11 ++- shadowsocks/obfsplugin/plain.py | 13 +++ shadowsocks/obfsplugin/verify_simple.py | 113 ++++++++++++++++++++++++ shadowsocks/tcprelay.py | 11 ++- 5 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 shadowsocks/obfsplugin/verify_simple.py diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 9d6d58ed..69eed0b1 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -23,12 +23,13 @@ import logging from shadowsocks import common -from shadowsocks.obfsplugin import plain, http_simple +from shadowsocks.obfsplugin import plain, http_simple, verify_simple method_supported = {} method_supported.update(plain.obfs) method_supported.update(http_simple.obfs) +method_supported.update(verify_simple.obfs) class Obfs(object): def __init__(self, method): @@ -49,15 +50,27 @@ def get_obfs(self, method): m = self._method_info return m[0](method) + def client_pre_encrypt(self, buf): + return self.obfs.client_pre_encrypt(buf) + def client_encode(self, buf): return self.obfs.client_encode(buf) def client_decode(self, buf): return self.obfs.client_decode(buf) + def client_post_decrypt(self, buf): + return self.obfs.client_post_decrypt(buf) + + def server_pre_encrypt(self, buf): + return self.obfs.server_pre_encrypt(buf) + def server_encode(self, buf): return self.obfs.server_encode(buf) def server_decode(self, buf): return self.obfs.server_decode(buf) + def server_post_decrypt(self, buf): + return self.obfs.server_post_decrypt(buf) + diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 6df2f2ef..9695136a 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -24,6 +24,8 @@ import binascii import base64 import datetime + +from shadowsocks.obfsplugin import plain from shadowsocks import common from shadowsocks.common import to_bytes, to_str, ord @@ -52,7 +54,7 @@ def match_begin(str1, str2): return True return False -class http_simple(object): +class http_simple(plain.plain): def __init__(self, method): self.method = method self.has_sent_header = False @@ -130,7 +132,7 @@ def server_decode(self, buf): self.has_recv_header = True return (buf, True, False) -class http2_simple(object): +class http2_simple(plain.plain): def __init__(self, method): self.method = method self.has_sent_header = False @@ -190,7 +192,7 @@ def server_decode(self, buf): self.has_recv_header = True return (buf, True, False) -class tls_simple(object): +class tls_simple(plain.plain): def __init__(self, method): self.method = method self.has_sent_header = False @@ -222,7 +224,7 @@ def server_decode(self, buf): # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) -class random_head(object): +class random_head(plain.plain): def __init__(self, method): self.method = method self.has_sent_header = False @@ -252,3 +254,4 @@ def server_decode(self, buf): return (buf, True, False) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) + diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index 5834f896..a40b9935 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -33,6 +33,9 @@ class plain(object): def __init__(self, method): self.method = method + def client_pre_encrypt(self, buf): + return buf + def client_encode(self, buf): return buf @@ -40,9 +43,19 @@ def client_decode(self, buf): # (buffer_to_recv, is_need_to_encode_and_send_back) return (buf, False) + def client_post_decrypt(self, buf): + return buf + + def server_pre_encrypt(self, buf): + return buf + def server_encode(self, buf): return buf def server_decode(self, buf): # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (buf, True, False) + + def server_post_decrypt(self, buf): + return buf + diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py new file mode 100644 index 00000000..ed22b628 --- /dev/null +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# +# Copyright 2015-2015 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import hashlib +import logging +import binascii +import base64 +import datetime +import struct + +from shadowsocks.obfsplugin import plain +from shadowsocks import common +from shadowsocks.common import to_bytes, to_str, ord + +def create_verify_obfs(method): + return verify_simple(method) + +obfs = { + 'verify_simple': (create_verify_obfs,), +} + +def match_begin(str1, str2): + if len(str1) >= len(str2): + if str1[:len(str2)] == str2: + return True + return False + +class verify_simple(plain.plain): + def __init__(self, method): + self.method = method + self.recv_buf = b'' + self.unit_len = 8100 + + def pack_data(self, buf): + if len(buf) == 0: + return b'' + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) + data = common.chr(len(rnd_data) + 1) + rnd_data + buf + data = struct.pack('>H', len(data) + 6) + data + crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff + data += struct.pack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_encode(self, buf): + return buf + + def client_decode(self, buf): + # (buffer_to_recv, is_need_to_encode_and_send_back) + return (buf, False) + + def client_post_decrypt(self, buf): + return buf + + def server_pre_encrypt(self, buf): + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_encode(self, buf): + return buf + + def server_decode(self, buf): + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (buf, True, False) + + def server_post_decrypt(self, buf): + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: + raise Exception('server_post_decrype data uncorrect CRC32') + + pos = common.ord(self.recv_buf[2]) + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + return out_buf + diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7e2b3050..6062996f 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -321,6 +321,7 @@ def _handel_protocol_error(self, client_address, ogn_data): def _handle_stage_connecting(self, data): if self._is_local: + data = self._obfs.client_pre_encrypt(data) data = self._encryptor.encrypt(data) data = self._obfs.client_encode(data) self._data_to_write_to_remote.append(data) @@ -419,6 +420,7 @@ def _handle_stage_addr(self, ogn_data, data): data = b'\x88' + struct.pack('>H', total_len) + chr(rnd_len) + (b' ' * (rnd_len - 1)) + data crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff data += struct.pack(' Date: Sun, 4 Oct 2015 22:59:45 +0800 Subject: [PATCH 050/487] obfs plugins compatible version --- config.json | 2 +- shadowsocks/obfsplugin/http_simple.py | 42 +++++++++++++------- shadowsocks/obfsplugin/verify_simple.py | 52 +++++++++++++++++++++++-- shadowsocks/tcprelay.py | 2 +- 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/config.json b/config.json index 11b34c23..2052b0d1 100644 --- a/config.json +++ b/config.json @@ -7,7 +7,7 @@ "password":"m", "timeout":300, "method":"aes-256-cfb", - "obfs":"http_simple", + "obfs":"http_simple_compatible", "fast_open": false, "workers": 1 } diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 9695136a..f9d04765 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -43,9 +43,13 @@ def create_random_head_obfs(method): obfs = { 'http_simple': (create_http_obfs,), + 'http_simple_compatible': (create_http_obfs,), 'http2_simple': (create_http2_obfs,), + 'http2_simple_compatible': (create_http2_obfs,), 'tls_simple': (create_tls_obfs,), + 'tls_simple_compatible': (create_tls_obfs,), 'random_head': (create_random_head_obfs,), + 'random_head_compatible': (create_random_head_obfs,), } def match_begin(str1, str2): @@ -95,6 +99,13 @@ def get_data_from_http_header(self, buf): return ret_buf return b'' + def not_match_return(self, buf): + self.has_sent_header = True + self.has_recv_header = True + if self.method == 'http_simple': + return (b'E', False, False) + return (buf, True, False) + def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) @@ -103,15 +114,11 @@ def server_decode(self, buf): if len(buf) > 10: if match_begin(buf, b'GET /') or match_begin(buf, b'POST /'): if len(buf) > 65536: - self.has_sent_header = True - self.has_recv_header = True self.recv_buffer = None - return (buf, True, False) + return self.not_match_return(buf) else: #not http header, run on original protocol - self.has_sent_header = True - self.has_recv_header = True self.recv_buffer = None - return (buf, True, False) + return self.not_match_return(buf) else: self.recv_buffer = buf return (b'', True, False) @@ -128,9 +135,7 @@ def server_decode(self, buf): else: self.recv_buffer = buf return (b'', True, False) - self.has_sent_header = True - self.has_recv_header = True - return (buf, True, False) + return self.not_match_return(buf) class http2_simple(plain.plain): def __init__(self, method): @@ -157,6 +162,13 @@ def server_encode(self, buf): self.has_sent_header = True return header + buf + def not_match_return(self, buf): + self.has_sent_header = True + self.has_recv_header = True + if self.method == 'http2_simple': + return (b'E', False, False) + return (buf, True, False) + def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) @@ -166,10 +178,8 @@ def server_decode(self, buf): if match_begin(buf, b'GET /'): pass else: #not http header, run on original protocol - self.has_sent_header = True - self.has_recv_header = True self.recv_buffer = None - return (buf, True, False) + return self.not_match_return(buf) else: self.recv_buffer = buf return (b'', True, False) @@ -188,9 +198,7 @@ def server_decode(self, buf): else: self.recv_buffer = buf return (b'', True, False) - self.has_sent_header = True - self.has_recv_header = True - return (buf, True, False) + return self.not_match_return(buf) class tls_simple(plain.plain): def __init__(self, method): @@ -220,6 +228,8 @@ def server_decode(self, buf): self.has_recv_header = True if not match_begin(buf, b'\x16\x03\x01'): self.has_sent_header = True + if self.method == 'tls_simple': + return (b'E', False, False) return (buf, True, False) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) @@ -251,6 +261,8 @@ def server_decode(self, buf): crc = binascii.crc32(buf) & 0xffffffff if crc != 0xffffffff: self.has_sent_header = True + if self.method == 'random_head': + return (b'E', False, False) return (buf, True, False) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index ed22b628..f89e7430 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -48,6 +48,8 @@ def __init__(self, method): self.method = method self.recv_buf = b'' self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False def pack_data(self, buf): if len(buf) == 0: @@ -75,7 +77,37 @@ def client_decode(self, buf): return (buf, False) def client_post_decrypt(self, buf): - return buf + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data uncorrect CRC32') + + pos = common.ord(self.recv_buf[2]) + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf def server_pre_encrypt(self, buf): ret = b'' @@ -93,21 +125,35 @@ def server_decode(self, buf): return (buf, True, False) def server_post_decrypt(self, buf): + if self.raw_trans: + return buf self.recv_buf += buf out_buf = b'' while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] if length >= 8192: - raise Exception('server_post_decrype data error') + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return b'E' + else: + raise Exception('server_post_decrype data error') if length > len(self.recv_buf): break if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: - raise Exception('server_post_decrype data uncorrect CRC32') + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return b'E' + else: + raise Exception('server_post_decrype data uncorrect CRC32') pos = common.ord(self.recv_buf[2]) + 2 out_buf += self.recv_buf[pos:length - 4] self.recv_buf = self.recv_buf[length:] + if out_buf: + self.decrypt_packet_num += 1 return out_buf diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 6062996f..99ac9b34 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -390,7 +390,7 @@ def _handle_stage_addr(self, ogn_data, data): if self._is_local: header_result = parse_header(data) else: - if FORCE_NEW_PROTOCOL and common.ord(data[0]) != 0x88: + if data is None or FORCE_NEW_PROTOCOL and common.ord(data[0]) != 0x88: data = self._handel_protocol_error(self._client_address, ogn_data) data = pre_parse_header(data) if data is None: From 36dc8f9463827f59cc99ed55b9b16cc8054d8272 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Fri, 9 Oct 2015 13:07:52 +0800 Subject: [PATCH 051/487] add verify_deflate --- shadowsocks/common.py | 2 +- shadowsocks/obfsplugin/verify_simple.py | 98 +++++++++++++++++++++++++ shadowsocks/tcprelay.py | 2 +- 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index cc88d5d1..2b11b9f0 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -160,7 +160,7 @@ def pre_parse_header(data): 'encryption method') return None data = data[rand_data_size + 3:] - elif datatype == 0x88: + elif datatype == 0x88 or (~datatype & 0xff) == 0x88: if len(data) <= 7 + 7: return None data_size = struct.unpack('>H', data[1:3])[0] diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index f89e7430..25bb55cf 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -25,6 +25,7 @@ import base64 import datetime import struct +import zlib from shadowsocks.obfsplugin import plain from shadowsocks import common @@ -33,8 +34,12 @@ def create_verify_obfs(method): return verify_simple(method) +def create_verify_deflate(method): + return verify_deflate(method) + obfs = { 'verify_simple': (create_verify_obfs,), + 'verify_deflate': (create_verify_deflate,), } def match_begin(str1, str2): @@ -157,3 +162,96 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return out_buf +class verify_deflate(plain.plain): + def __init__(self, method): + self.method = method + self.recv_buf = b'' + self.unit_len = 32700 + self.decrypt_packet_num = 0 + self.raw_trans = False + + def pack_data(self, buf): + if len(buf) == 0: + return b'' + data = zlib.compress(buf) + data = struct.pack('>H', len(data)) + data[2:] + return data + + def client_pre_encrypt(self, buf): + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_encode(self, buf): + return buf + + def client_decode(self, buf): + # (buffer_to_recv, is_need_to_encode_and_send_back) + return (buf, False) + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 32768: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + out_buf += zlib.decompress(b'x\x9c' + self.recv_buf[2:length]) + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf + + def server_pre_encrypt(self, buf): + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_encode(self, buf): + return buf + + def server_decode(self, buf): + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (buf, True, False) + + def server_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 32768: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + out_buf += zlib.decompress(b'x\x9c' + self.recv_buf[2:length]) + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf + diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 99ac9b34..2e42d595 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -390,7 +390,7 @@ def _handle_stage_addr(self, ogn_data, data): if self._is_local: header_result = parse_header(data) else: - if data is None or FORCE_NEW_PROTOCOL and common.ord(data[0]) != 0x88: + if data is None or FORCE_NEW_PROTOCOL and common.ord(data[0]) != 0x88 and (~common.ord(data[0]) & 0xff) != 0x88: data = self._handel_protocol_error(self._client_address, ogn_data) data = pre_parse_header(data) if data is None: From c73127d858979a87d925565ddb0749d4d24fea8c Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Sat, 10 Oct 2015 19:47:50 +0800 Subject: [PATCH 052/487] insert a reload step main thread run 'thread_db' directly insert a 'ulimit' command before run the server --- db_transfer.py | 3 +++ run.sh | 5 +++-- server.py | 19 +++++++++++++------ stop.sh | 2 +- tail.sh | 2 +- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index ae65939c..61a72550 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -78,9 +78,11 @@ def pull_db_all_user(): #数据库所有用户信息 try: import switchrule + reload(switchrule) keys = switchrule.getKeys() except Exception as e: keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] + reload(cymysql) conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') cur = conn.cursor() @@ -102,6 +104,7 @@ def del_server_out_of_bound_safe(last_rows, rows): #需要动态载入switchrule，以便实时修改规则 try: import switchrule + reload(switchrule) except Exception as e: logging.error('load switchrule.py fail') cur_servers = {} diff --git a/run.sh b/run.sh index 2925a5ec..8fceb971 100644 --- a/run.sh +++ b/run.sh @@ -1,5 +1,6 @@ #!/bin/bash cd `dirname $0` -eval $(ps -ef | grep "[0-9] python server\\.py" | awk '{print "kill "$2}') -nohup python server.py >> ssserver.log 2>&1 & +eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') +ulimit -n 131072 +nohup python server.py m>> ssserver.log 2>&1 & diff --git a/server.py b/server.py index 585380f9..9bd538e6 100644 --- a/server.py +++ b/server.py @@ -7,6 +7,7 @@ import server_pool import db_transfer +from shadowsocks import shell #def test(): # thread.start_new_thread(DbTransfer.thread_db, ()) @@ -19,10 +20,16 @@ def __init__(self): def run(self): db_transfer.DbTransfer.thread_db() +def main(): + shell.check_python() + if len(sys.argv) <= 3: + db_transfer.DbTransfer.thread_db() + else: + thread = MainThread() + thread.start() + while True: + time.sleep(99999) + if __name__ == '__main__': - #server_pool.ServerPool.get_instance() - #server_pool.ServerPool.get_instance().new_server(2333, '2333') - thread = MainThread() - thread.start() - while True: - time.sleep(99999) + main() + diff --git a/stop.sh b/stop.sh index 2fa55cf9..c726221a 100644 --- a/stop.sh +++ b/stop.sh @@ -1,3 +1,3 @@ #!/bin/bash -eval $(ps -ef | grep "[0-9] python server\\.py" | awk '{print "kill "$2}') +eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') diff --git a/tail.sh b/tail.sh index aa371393..f36f605e 100644 --- a/tail.sh +++ b/tail.sh @@ -1,3 +1,3 @@ #!/bin/bash - +cd `dirname $0` tail -f ssserver.log From 6d963cea44bd469a506a43380498a6ac7e0765e8 Mon Sep 17 00:00:00 2001 From: falseen Date: Sun, 11 Oct 2015 22:38:46 +0800 Subject: [PATCH 053/487] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B7=B7=E6=B7=86?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E5=8F=82=E6=95=B0"-o"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- shadowsocks/shell.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index e5f5bb52..4c747acd 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -130,11 +130,11 @@ def get_config(is_local): logging.basicConfig(level=logging.INFO, format='%(levelname)-s: %(message)s') if is_local: - shortopts = 'hd:s:b:p:k:l:m:c:t:vq' + shortopts = 'hd:s:b:p:k:l:m:o:c:t:vq' longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'user=', 'version'] else: - shortopts = 'hd:s:p:k:m:c:t:vq' + shortopts = 'hd:s:p:k:m:o:c:t:vq' longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'workers=', 'forbidden-ip=', 'user=', 'manager-address=', 'version'] try: @@ -168,6 +168,8 @@ def get_config(is_local): config['server'] = to_str(value) elif key == '-m': config['method'] = to_str(value) + elif key == '-o': + config['obfs'] = to_str(value) elif key == '-b': config['local_address'] = to_str(value) elif key == '-v': @@ -286,6 +288,7 @@ def print_local_help(): -l LOCAL_PORT local port, default: 1080 -k PASSWORD password -m METHOD encryption method, default: aes-256-cfb + -o OBFS obfsplugin, default: http_simple -t TIMEOUT timeout in seconds, default: 300 --fast-open use TCP_FASTOPEN, requires Linux 3.7+ @@ -315,6 +318,7 @@ def print_server_help(): -p SERVER_PORT server port, default: 8388 -k PASSWORD password -m METHOD encryption method, default: aes-256-cfb + -o OBFS obfsplugin, default: http_simple -t TIMEOUT timeout in seconds, default: 300 --fast-open use TCP_FASTOPEN, requires Linux 3.7+ --workers WORKERS number of workers, available on Unix/Linux From 612c1e2ab3be39a497a758c816bf298abc89db04 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 20 Oct 2015 16:25:06 +0800 Subject: [PATCH 054/487] add obfs_param catch TCP&UDP exceptions verify_simple&verify_deflate support sub obfs --- config.json | 15 +++--- server.py | 2 +- shadowsocks/obfs.py | 21 +++++--- shadowsocks/obfsplugin/http_simple.py | 4 +- shadowsocks/obfsplugin/plain.py | 9 +++- shadowsocks/obfsplugin/verify_simple.py | 71 ++++++++++++++++++++++--- shadowsocks/shell.py | 1 + shadowsocks/tcprelay.py | 39 ++++++++++++-- shadowsocks/udprelay.py | 23 +++++++- 9 files changed, 154 insertions(+), 31 deletions(-) diff --git a/config.json b/config.json index 2052b0d1..d9a4c4be 100644 --- a/config.json +++ b/config.json @@ -1,13 +1,14 @@ { - "server":"0.0.0.0", + "server": "0.0.0.0", "server_ipv6": "::", - "server_port":8388, + "server_port": 8388, "local_address": "127.0.0.1", - "local_port":1080, - "password":"m", - "timeout":300, - "method":"aes-256-cfb", - "obfs":"http_simple_compatible", + "local_port": 1080, + "password": "m", + "timeout": 300, + "method": "aes-256-cfb", + "obfs": "http_simple_compatible", + "obfs_param": "", "fast_open": false, "workers": 1 } diff --git a/server.py b/server.py index 9bd538e6..2030397d 100644 --- a/server.py +++ b/server.py @@ -22,7 +22,7 @@ def run(self): def main(): shell.check_python() - if len(sys.argv) <= 3: + if True: db_transfer.DbTransfer.thread_db() else: thread = MainThread() diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 69eed0b1..92de548f 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -27,19 +27,28 @@ method_supported = {} -method_supported.update(plain.obfs) -method_supported.update(http_simple.obfs) -method_supported.update(verify_simple.obfs) +method_supported.update(plain.obfs_map) +method_supported.update(http_simple.obfs_map) +method_supported.update(verify_simple.obfs_map) -class Obfs(object): +class server_info(object): + def __init__(self, data): + self.data = data + +class obfs(object): def __init__(self, method): self.method = method self._method_info = self.get_method_info(method) if self._method_info: self.obfs = self.get_obfs(method) else: - logging.error('method %s not supported' % method) - sys.exit(1) + raise Exception('method %s not supported' % method) + + def init_data(self): + return self.obfs.init_data() + + def set_server_info(self, server_info): + return self.obfs.set_server_info(server_info) def get_method_info(self, method): method = method.lower() diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index f9d04765..75e92948 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -25,8 +25,8 @@ import base64 import datetime -from shadowsocks.obfsplugin import plain from shadowsocks import common +from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord def create_http_obfs(method): @@ -41,7 +41,7 @@ def create_tls_obfs(method): def create_random_head_obfs(method): return random_head(method) -obfs = { +obfs_map = { 'http_simple': (create_http_obfs,), 'http_simple_compatible': (create_http_obfs,), 'http2_simple': (create_http2_obfs,), diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index a40b9935..cc602fae 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -25,13 +25,20 @@ def create_obfs(method): return plain(method) -obfs = { +obfs_map = { 'plain': (create_obfs,), } class plain(object): def __init__(self, method): self.method = method + self.server_info = None + + def init_data(self): + return b'' + + def set_server_info(self, server_info): + self.server_info = server_info def client_pre_encrypt(self, buf): return buf diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 25bb55cf..ab9f2a5e 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -27,8 +27,9 @@ import struct import zlib -from shadowsocks.obfsplugin import plain +import shadowsocks from shadowsocks import common +from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord def create_verify_obfs(method): @@ -37,7 +38,7 @@ def create_verify_obfs(method): def create_verify_deflate(method): return verify_deflate(method) -obfs = { +obfs_map = { 'verify_simple': (create_verify_obfs,), 'verify_deflate': (create_verify_deflate,), } @@ -48,6 +49,10 @@ def match_begin(str1, str2): return True return False +class sub_encode_obfs(object): + def __init__(self): + self.sub_obfs = None + class verify_simple(plain.plain): def __init__(self, method): self.method = method @@ -55,6 +60,26 @@ def __init__(self, method): self.unit_len = 8100 self.decrypt_packet_num = 0 self.raw_trans = False + self.sub_obfs = None + + def init_data(self): + return sub_encode_obfs() + + def set_server_info(self, server_info): + try: + if server_info.param: + self.sub_obfs = shadowsocks.obfs.obfs(server_info.param) + if server_info.data.sub_obfs is None: + server_info.data.sub_obfs = self.sub_obfs.init_data() + _server_info = shadowsocks.obfs.server_info(server_info.data.sub_obfs) + _server_info.host = server_info.host + _server_info.port = server_info.port + _server_info.tcp_mss = server_info.tcp_mss + _server_info.param = '' + self.sub_obfs.set_server_info(_server_info) + except Exception as e: + shadowsocks.shell.print_exception(e) + self.server_info = server_info def pack_data(self, buf): if len(buf) == 0: @@ -75,10 +100,13 @@ def client_pre_encrypt(self, buf): return ret def client_encode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.client_encode(buf) return buf def client_decode(self, buf): - # (buffer_to_recv, is_need_to_encode_and_send_back) + if self.sub_obfs is not None: + return self.sub_obfs.client_decode(buf) return (buf, False) def client_post_decrypt(self, buf): @@ -123,10 +151,13 @@ def server_pre_encrypt(self, buf): return ret def server_encode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.server_encode(buf) return buf def server_decode(self, buf): - # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + if self.sub_obfs is not None: + return self.sub_obfs.server_decode(buf) return (buf, True, False) def server_post_decrypt(self, buf): @@ -169,6 +200,26 @@ def __init__(self, method): self.unit_len = 32700 self.decrypt_packet_num = 0 self.raw_trans = False + self.sub_obfs = None + + def init_data(self): + return sub_encode_obfs() + + def set_server_info(self, server_info): + try: + if server_info.param: + self.sub_obfs = shadowsocks.obfs.obfs(server_info.param) + if server_info.data.sub_obfs is None: + server_info.data.sub_obfs = self.sub_obfs.init_data() + _server_info = shadowsocks.obfs.server_info(server_info.data.sub_obfs) + _server_info.host = server_info.host + _server_info.port = server_info.port + _server_info.tcp_mss = server_info.tcp_mss + _server_info.param = '' + self.sub_obfs.set_server_info(_server_info) + except Exception as e: + shadowsocks.shell.print_exception(e) + self.server_info = server_info def pack_data(self, buf): if len(buf) == 0: @@ -186,10 +237,13 @@ def client_pre_encrypt(self, buf): return ret def client_encode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.client_encode(buf) return buf def client_decode(self, buf): - # (buffer_to_recv, is_need_to_encode_and_send_back) + if self.sub_obfs is not None: + return self.sub_obfs.client_decode(buf) return (buf, False) def client_post_decrypt(self, buf): @@ -225,10 +279,13 @@ def server_pre_encrypt(self, buf): return ret def server_encode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.server_encode(buf) return buf def server_decode(self, buf): - # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + if self.sub_obfs is not None: + return self.sub_obfs.server_decode(buf) return (buf, True, False) def server_post_decrypt(self, buf): @@ -248,7 +305,7 @@ def server_post_decrypt(self, buf): if length > len(self.recv_buf): break - out_buf += zlib.decompress(b'x\x9c' + self.recv_buf[2:length]) + out_buf += zlib.decompress(b'\x78\x9c' + self.recv_buf[2:length]) self.recv_buf = self.recv_buf[length:] if out_buf: diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index e5f5bb52..7327b34c 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -217,6 +217,7 @@ def get_config(is_local): config['password'] = to_bytes(config.get('password', b'')) config['method'] = to_str(config.get('method', 'aes-256-cfb')) config['obfs'] = to_str(config.get('obfs', 'plain')) + config['obfs_param'] = to_str(config.get('obfs_param', '')) config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) config['fast_open'] = config.get('fast_open', False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 2e42d595..2bb973a9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -117,7 +117,16 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._encryptor = encrypt.Encryptor(config['password'], config['method']) self._encrypt_correct = True - self._obfs = obfs.Obfs(config['obfs']) + self._obfs = obfs.obfs(config['obfs']) + if server.obfs_data is None: + server.obfs_data = self._obfs.init_data() + server_info = obfs.server_info(server.obfs_data) + server_info.host = config['server'] + server_info.port = server._listen_port + server_info.tcp_mss = 1440 + server_info.param = config['obfs_param'] + self._obfs.set_server_info(server_info) + self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] @@ -139,6 +148,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._server) self.last_activity = 0 self._update_activity() + self._server.add_connection(1) def __hash__(self): # default __hash__ is id / 16 @@ -276,6 +286,10 @@ def _write_to_sock(self, data, sock): shell.print_exception(e) self.destroy() return False + except Exception as e: + shell.print_exception(e) + self.destroy() + return False if uncomplete: if sock == self._local_sock: self._data_to_write_to_local.append(data) @@ -467,7 +481,7 @@ def _create_remote_socket(self, ip, port): remote_sock.setblocking(False) if self._remote_udp: - pass + remote_sock_v6.setblocking(False) else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) return remote_sock @@ -515,7 +529,9 @@ def _handle_dns_resolved(self, result, error): except (OSError, IOError) as e: if eventloop.errno_from_exception(e) == \ errno.EINPROGRESS: - pass + pass # always goto here + else: + raise e self._loop.add(remote_sock, eventloop.POLL_ERR | eventloop.POLL_OUT, self._server) @@ -721,13 +737,19 @@ def destroy(self): logging.debug('destroy') if self._remote_sock: logging.debug('destroying remote') - self._loop.remove(self._remote_sock) + try: + self._loop.remove(self._remote_sock) + except Exception as e: + pass del self._fd_to_handlers[self._remote_sock.fileno()] self._remote_sock.close() self._remote_sock = None if self._remote_sock_v6: logging.debug('destroying remote') - self._loop.remove(self._remote_sock_v6) + try: + self._loop.remove(self._remote_sock_v6) + except Exception as e: + pass del self._fd_to_handlers[self._remote_sock_v6.fileno()] self._remote_sock_v6.close() self._remote_sock_v6 = None @@ -739,6 +761,7 @@ def destroy(self): self._local_sock = None self._dns_resolver.remove_callback(self._handle_dns_resolved) self._server.remove_handler(self) + self._server.add_connection(-1) class TCPRelay(object): def __init__(self, config, dns_resolver, is_local, stat_callback=None): @@ -750,6 +773,8 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._fd_to_handlers = {} self.server_transfer_ul = 0 self.server_transfer_dl = 0 + self.server_connections = 0 + self.obfs_data = None self._timeout = config['timeout'] self._timeouts = [] # a list for all the handlers @@ -802,6 +827,10 @@ def remove_handler(self, handler): self._timeouts[index] = None del self._handler_to_timeouts[hash(handler)] + def add_connection(self, val): + self.server_connections += val + logging.debug('server port %5d connections = %d' % (self._listen_port, self.server_connections,)) + def update_activity(self, handler, data_len): if data_len and self._stat_callback: self._stat_callback(self._listen_port, data_len) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index d7626c9c..d3696d9e 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1095,6 +1095,12 @@ def _handle_server(self): self._sockets.add(client.fileno()) self._eventloop.add(client, eventloop.POLL_IN, self) + logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) + + logging.info('UDP data to %s:%d from %s:%d' % + (common.to_str(server_addr), server_port, + r_addr[0], r_addr[1])) + if self._is_local: data = encrypt.encrypt_all(self._password, self._method, 1, data) if not data: @@ -1254,11 +1260,21 @@ def handle_event(self, sock, fd, event): if sock == self._server_socket: if event & eventloop.POLL_ERR: logging.error('UDP server_socket err') - self._handle_server() + try: + self._handle_server() + except Exception as e: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() elif sock and (fd in self._sockets): if event & eventloop.POLL_ERR: logging.error('UDP client_socket err') - self._handle_client(sock) + try: + self._handle_client(sock) + except Exception as e: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() else: if sock: handler = self._fd_to_handlers.get(fd, None) @@ -1275,7 +1291,10 @@ def handle_periodic(self): for sock in self._sockets: sock.close() logging.info('closed UDP port %d', self._listen_port) + before_sweep_size = len(self._sockets) self._cache.sweep() + if before_sweep_size != len(self._sockets): + logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) self._client_fd_to_server_addr.sweep() self._sweep_timeout() From 4551d5c268a82f6f6f6af605c4b603ba69be2759 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 20 Oct 2015 17:01:37 +0800 Subject: [PATCH 055/487] shorter code --- shadowsocks/obfsplugin/verify_simple.py | 109 +++++++++--------------- 1 file changed, 40 insertions(+), 69 deletions(-) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index ab9f2a5e..10ba8f70 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -53,13 +53,10 @@ class sub_encode_obfs(object): def __init__(self): self.sub_obfs = None -class verify_simple(plain.plain): +class verify_base(plain.plain): def __init__(self, method): + super(verify_base, self).__init__(method) self.method = method - self.recv_buf = b'' - self.unit_len = 8100 - self.decrypt_packet_num = 0 - self.raw_trans = False self.sub_obfs = None def init_data(self): @@ -68,19 +65,53 @@ def init_data(self): def set_server_info(self, server_info): try: if server_info.param: - self.sub_obfs = shadowsocks.obfs.obfs(server_info.param) + sub_param = '' + param_list = server_info.param.split(',', 1) + if len(param_list) > 1: + self.sub_obfs = shadowsocks.obfs.obfs(param_list[0]) + sub_param = param_list[1] + else: + self.sub_obfs = shadowsocks.obfs.obfs(server_info.param) if server_info.data.sub_obfs is None: server_info.data.sub_obfs = self.sub_obfs.init_data() _server_info = shadowsocks.obfs.server_info(server_info.data.sub_obfs) _server_info.host = server_info.host _server_info.port = server_info.port _server_info.tcp_mss = server_info.tcp_mss - _server_info.param = '' + _server_info.param = sub_param self.sub_obfs.set_server_info(_server_info) except Exception as e: shadowsocks.shell.print_exception(e) self.server_info = server_info + def client_encode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.client_encode(buf) + return buf + + def client_decode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.client_decode(buf) + return (buf, False) + + def server_encode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.server_encode(buf) + return buf + + def server_decode(self, buf): + if self.sub_obfs is not None: + return self.sub_obfs.server_decode(buf) + return (buf, True, False) + +class verify_simple(verify_base): + def __init__(self, method): + super(verify_simple, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False + def pack_data(self, buf): if len(buf) == 0: return b'' @@ -99,16 +130,6 @@ def client_pre_encrypt(self, buf): ret += self.pack_data(buf) return ret - def client_encode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.client_encode(buf) - return buf - - def client_decode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.client_decode(buf) - return (buf, False) - def client_post_decrypt(self, buf): if self.raw_trans: return buf @@ -150,16 +171,6 @@ def server_pre_encrypt(self, buf): ret += self.pack_data(buf) return ret - def server_encode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.server_encode(buf) - return buf - - def server_decode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.server_decode(buf) - return (buf, True, False) - def server_post_decrypt(self, buf): if self.raw_trans: return buf @@ -193,33 +204,13 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return out_buf -class verify_deflate(plain.plain): +class verify_deflate(verify_base): def __init__(self, method): - self.method = method + super(verify_deflate, self).__init__(method) self.recv_buf = b'' self.unit_len = 32700 self.decrypt_packet_num = 0 self.raw_trans = False - self.sub_obfs = None - - def init_data(self): - return sub_encode_obfs() - - def set_server_info(self, server_info): - try: - if server_info.param: - self.sub_obfs = shadowsocks.obfs.obfs(server_info.param) - if server_info.data.sub_obfs is None: - server_info.data.sub_obfs = self.sub_obfs.init_data() - _server_info = shadowsocks.obfs.server_info(server_info.data.sub_obfs) - _server_info.host = server_info.host - _server_info.port = server_info.port - _server_info.tcp_mss = server_info.tcp_mss - _server_info.param = '' - self.sub_obfs.set_server_info(_server_info) - except Exception as e: - shadowsocks.shell.print_exception(e) - self.server_info = server_info def pack_data(self, buf): if len(buf) == 0: @@ -236,16 +227,6 @@ def client_pre_encrypt(self, buf): ret += self.pack_data(buf) return ret - def client_encode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.client_encode(buf) - return buf - - def client_decode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.client_decode(buf) - return (buf, False) - def client_post_decrypt(self, buf): if self.raw_trans: return buf @@ -278,16 +259,6 @@ def server_pre_encrypt(self, buf): ret += self.pack_data(buf) return ret - def server_encode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.server_encode(buf) - return buf - - def server_decode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.server_decode(buf) - return (buf, True, False) - def server_post_decrypt(self, buf): if self.raw_trans: return buf From eec312cc4cdfd12a514d9c735c38fcf1907f03e1 Mon Sep 17 00:00:00 2001 From: breakwa11 Date: Tue, 20 Oct 2015 18:47:31 +0800 Subject: [PATCH 056/487] bump 2.6.11 --- CHANGES | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES b/CHANGES index 4db142a6..5cdbccdc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +2.6.11 2015-10-20 +- Obfs plugin +- Obfs parameters +- UDP over TCP +- TCP over UDP (experimental) +- Fix socket leaks +- Catch abnormal UDP package + 2.6.10 2015-06-08 - Optimize LRU cache - Refine logging From 79260ff4ab63d851a434971b4f0ce614cd71b299 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 21 Oct 2015 14:53:14 +0800 Subject: [PATCH 057/487] check duplicate port --- db_transfer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index 61a72550..e96819a0 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -116,7 +116,12 @@ def del_server_out_of_bound_safe(last_rows, rows): port = row['port'] passwd = row['passwd'] - cur_servers[port] = passwd + + if port not in cur_servers: + cur_servers[port] = passwd + else: + logging.error('more than one user use the same port [%s]' % (port,)) + continue if ServerPool.get_instance().server_is_run(port) > 0: if not allow: From 041a5c1046e08c9f2b9b6b5a75e04d63be29ac64 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 21 Oct 2015 22:02:22 +0800 Subject: [PATCH 058/487] optimize decrypt first recv size < iv_len will cause decrypt error --- db_transfer.py | 3 ++- server_pool.py | 4 ++-- shadowsocks/encrypt.py | 21 +++++++++++++-------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index e96819a0..ba6e3eab 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -8,6 +8,7 @@ from server_pool import ServerPool import Config import traceback +from shadowsocks import common class DbTransfer(object): @@ -115,7 +116,7 @@ def del_server_out_of_bound_safe(last_rows, rows): allow = False port = row['port'] - passwd = row['passwd'] + passwd = common.to_bytes(row['passwd']) if port not in cur_servers: cur_servers[port] = passwd diff --git a/server_pool.py b/server_pool.py index 77d300d7..4b4663c0 100644 --- a/server_pool.py +++ b/server_pool.py @@ -187,9 +187,9 @@ def cb_del_server(self, port): if 'server_ipv6' in self.config: if port not in self.tcp_ipv6_servers_pool: - logging.info("stopped server at %s:%d already stop" % (self.config['server_ipv6'], port)) + logging.info("stopped server at [%s]:%d already stop" % (self.config['server_ipv6'], port)) else: - logging.info("stopped server at %s:%d" % (self.config['server_ipv6'], port)) + logging.info("stopped server at [%s]:%d" % (self.config['server_ipv6'], port)) try: self.tcp_ipv6_servers_pool[port].close(False) del self.tcp_ipv6_servers_pool[port] diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 834b18ce..b7981b48 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -77,6 +77,7 @@ def __init__(self, key, method): self.iv = None self.iv_sent = False self.cipher_iv = b'' + self.iv_buf = b'' self.decipher = None method = method.lower() self._method_info = self.get_method_info(method) @@ -122,16 +123,20 @@ def encrypt(self, buf): def decrypt(self, buf): if len(buf) == 0: return buf - if self.decipher is None: - decipher_iv_len = self._method_info[1] - decipher_iv = buf[:decipher_iv_len] + if self.decipher is not None: #optimize + return self.decipher.update(buf) + + decipher_iv_len = self._method_info[1] + if len(self.iv_buf) <= decipher_iv_len: + self.iv_buf += buf + if len(self.iv_buf) > decipher_iv_len: + decipher_iv = self.iv_buf[:decipher_iv_len] self.decipher = self.get_cipher(self.key, self.method, 0, iv=decipher_iv) - buf = buf[decipher_iv_len:] - if len(buf) == 0: - return buf - return self.decipher.update(buf) - + buf = self.iv_buf[decipher_iv_len:] + return self.decipher.update(buf) + else: + return b'' def encrypt_all(password, method, op, data): result = [] From 3f66ec443f1f0b93993f2676e6cd4eb00fb19b30 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 21 Oct 2015 22:37:32 +0800 Subject: [PATCH 059/487] remove FORCE_NEW_PROTOCOL --- shadowsocks/tcprelay.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 2bb973a9..a51830a5 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -30,11 +30,8 @@ from shadowsocks import encrypt, obfs, eventloop, shell, common from shadowsocks.common import pre_parse_header, parse_header -# set it 'False' to use both new protocol and the original shadowsocks protocal -# set it 'True' to use new protocol ONLY, to avoid GFW detecting -FORCE_NEW_PROTOCOL = False # set it 'True' if run as a local client and connect to a server which support new protocol -CLIENT_NEW_PROTOCOL = False +CLIENT_NEW_PROTOCOL = False #deprecated # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 @@ -404,8 +401,6 @@ def _handle_stage_addr(self, ogn_data, data): if self._is_local: header_result = parse_header(data) else: - if data is None or FORCE_NEW_PROTOCOL and common.ord(data[0]) != 0x88 and (~common.ord(data[0]) & 0xff) != 0x88: - data = self._handel_protocol_error(self._client_address, ogn_data) data = pre_parse_header(data) if data is None: data = self._handel_protocol_error(self._client_address, ogn_data) From 126afea524f7c56b25c56553aee31c79bc20e05d Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 22 Oct 2015 10:41:40 +0800 Subject: [PATCH 060/487] delete buffer --- shadowsocks/encrypt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index b7981b48..d3b27527 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -134,6 +134,7 @@ def decrypt(self, buf): self.decipher = self.get_cipher(self.key, self.method, 0, iv=decipher_iv) buf = self.iv_buf[decipher_iv_len:] + del self.iv_buf return self.decipher.update(buf) else: return b'' From 0ba76b666d804436dd8383815f2221309cad2590 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 22 Oct 2015 15:55:38 +0800 Subject: [PATCH 061/487] not redirect host to chn web add auth_simple --- shadowsocks/obfsplugin/verify_simple.py | 217 +++++++++++++++++++++++- shadowsocks/tcprelay.py | 2 +- 2 files changed, 216 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 10ba8f70..fde73faa 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -23,7 +23,9 @@ import logging import binascii import base64 +import time import datetime +import random import struct import zlib @@ -38,9 +40,13 @@ def create_verify_obfs(method): def create_verify_deflate(method): return verify_deflate(method) +def create_auth_obfs(method): + return auth_simple(method) + obfs_map = { 'verify_simple': (create_verify_obfs,), 'verify_deflate': (create_verify_deflate,), + 'auth_simple': (create_auth_obfs,), } def match_begin(str1, str2): @@ -49,7 +55,7 @@ def match_begin(str1, str2): return True return False -class sub_encode_obfs(object): +class obfs_verify_data(object): def __init__(self): self.sub_obfs = None @@ -60,7 +66,7 @@ def __init__(self, method): self.sub_obfs = None def init_data(self): - return sub_encode_obfs() + return obfs_verify_data() def set_server_info(self, server_info): try: @@ -283,3 +289,210 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return out_buf +class client_queue(object): + def __init__(self, begin_id): + self.front = begin_id + self.back = begin_id + self.alloc = {} + self.enable = True + self.last_update = time.time() + + def update(self): + self.last_update = time.time() + + def is_active(self): + return time.time() - self.last_update < 60 * 3 + + def re_enable(self, connection_id): + self.enable = True + self.alloc = {} + self.front = connection_id + self.back = connection_id + + def insert(self, connection_id): + self.update() + if not self.enable: + return False + if connection_id < self.front: + return False + if not self.is_active(): + self.re_enable(connection_id) + if connection_id > self.front + 0x4000: + return False + if connection_id in self.alloc: + return False + if self.back <= connection_id: + self.back = connection_id + 1 + self.alloc[connection_id] = 1 + while (self.front in self.alloc) or self.front + 0x1000 < self.back: + if self.front in self.alloc: + del self.alloc[self.front] + self.front += 1 + return True + +class obfs_auth_data(object): + def __init__(self): + self.sub_obfs = None + self.client_id = {} + + def update(self, client_id, connection_id): + if client_id in self.client_id: + self.client_id[client_id].update() + + def insert(self, client_id, connection_id): + max_client = 16 + if client_id not in self.client_id or not self.client_id[client_id].enable: + active = 0 + for c_id in self.client_id: + if self.client_id[c_id].is_active(): + active += 1 + if active >= max_client: + return False + + if len(self.client_id) < max_client: + if client_id not in self.client_id: + self.client_id[client_id] = client_queue(connection_id) + else: + self.client_id[client_id].re_enable(connection_id) + return self.client_id[client_id].insert(connection_id) + keys = self.client_id.keys() + random.shuffle(keys) + for c_id in keys: + if not self.client_id[c_id].is_active() and self.client_id[c_id].enable: + if len(self.client_id) >= 256: + del self.client_id[c_id] + else: + self.client_id[c_id].enable = False + if client_id not in self.client_id: + self.client_id[client_id] = client_queue(connection_id) + else: + self.client_id[client_id].re_enable(connection_id) + return self.client_id[client_id].insert(connection_id) + return False + else: + return self.client_id[client_id].insert(connection_id) + +class auth_simple(verify_base): + def __init__(self, method): + super(auth_simple, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + + def init_data(self): + return obfs_auth_data() + + def pack_data(self, buf): + if len(buf) == 0: + return b'' + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) + data = common.chr(len(rnd_data) + 1) + rnd_data + buf + data = struct.pack('>H', len(data) + 6) + data + crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff + data += struct.pack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data uncorrect CRC32') + + pos = common.ord(self.recv_buf[2]) + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf + + def server_pre_encrypt(self, buf): + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return b'E' + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return b'E' + else: + raise Exception('server_post_decrype data uncorrect CRC32') + + pos = common.ord(self.recv_buf[2]) + 2 + out_buf += self.recv_buf[pos:length - 4] + if not self.has_recv_header: + if len(out_buf) < 8: + self.raw_trans = True + self.recv_buf = b'' + return b'E' + client_id = struct.unpack(' Date: Fri, 23 Oct 2015 15:17:30 +0800 Subject: [PATCH 062/487] optimize code add logs add timestamp --- shadowsocks/common.py | 10 +++++++++ shadowsocks/obfsplugin/http_simple.py | 15 ++++++-------- shadowsocks/obfsplugin/verify_simple.py | 27 ++++++++++++++++++++----- shadowsocks/server.py | 5 +++-- 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 2b11b9f0..7f306ea8 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -54,6 +54,16 @@ def to_str(s): return s.decode('utf-8') return s +def int32(x): + if x > 0xFFFFFFFF or x < 0: + x &= 0xFFFFFFFF + if x > 0x7FFFFFFF: + x = int(0x100000000 - x) + if x < 0x80000000: + return -x + else: + return -2147483648 + return x def inet_ntop(family, ipstr): if family == socket.AF_INET: diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 75e92948..637015fd 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -110,17 +110,19 @@ def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) - buf = self.recv_buffer + buf + self.recv_buffer += buf + buf = self.recv_buffer if len(buf) > 10: if match_begin(buf, b'GET /') or match_begin(buf, b'POST /'): if len(buf) > 65536: self.recv_buffer = None + logging.warn('http_simple: over size') return self.not_match_return(buf) else: #not http header, run on original protocol self.recv_buffer = None + logging.debug('http_simple: not match begin') return self.not_match_return(buf) else: - self.recv_buffer = buf return (b'', True, False) datas = buf.split(b'\r\n\r\n', 1) @@ -130,12 +132,9 @@ def server_decode(self, buf): if len(ret_buf) >= 15: self.has_recv_header = True return (ret_buf, True, False) - self.recv_buffer = buf return (b'', True, False) else: - self.recv_buffer = buf return (b'', True, False) - return self.not_match_return(buf) class http2_simple(plain.plain): def __init__(self, method): @@ -173,7 +172,8 @@ def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) - buf = self.recv_buffer + buf + self.recv_buffer += buf + buf = self.recv_buffer if len(buf) > 10: if match_begin(buf, b'GET /'): pass @@ -181,7 +181,6 @@ def server_decode(self, buf): self.recv_buffer = None return self.not_match_return(buf) else: - self.recv_buffer = buf return (b'', True, False) datas = buf.split(b'\r\n\r\n', 1) @@ -193,10 +192,8 @@ def server_decode(self, buf): ret_buf += datas[1] self.has_recv_header = True return (ret_buf, True, False) - self.recv_buffer = buf return (b'', True, False) else: - self.recv_buffer = buf return (b'', True, False) return self.not_match_return(buf) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index fde73faa..c162c064 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -312,14 +312,18 @@ def re_enable(self, connection_id): def insert(self, connection_id): self.update() if not self.enable: + logging.warn('auth_simple: not enable') return False if connection_id < self.front: + logging.warn('auth_simple: duplicate id') return False if not self.is_active(): self.re_enable(connection_id) if connection_id > self.front + 0x4000: + logging.warn('auth_simple: wrong id') return False if connection_id in self.alloc: + logging.warn('auth_simple: duplicate id 2') return False if self.back <= connection_id: self.back = connection_id + 1 @@ -347,6 +351,7 @@ def insert(self, client_id, connection_id): if self.client_id[c_id].is_active(): active += 1 if active >= max_client: + logging.warn('auth_simple: max active clients exceeded') return False if len(self.client_id) < max_client: @@ -368,6 +373,7 @@ def insert(self, client_id, connection_id): else: self.client_id[client_id].re_enable(connection_id) return self.client_id[client_id].insert(connection_id) + logging.warn('auth_simple: no inactive client [assert]') return False else: return self.client_id[client_id].insert(connection_id) @@ -457,6 +463,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: + logging.info('auth_simple: over size') return b'E' else: raise Exception('server_post_decrype data error') @@ -464,6 +471,7 @@ def server_post_decrypt(self, buf): break if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: + logging.info('auth_simple: crc32 error, data %s' % (binascii.hexlify(self.recv_buf[:length]),)) self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: @@ -474,20 +482,29 @@ def server_post_decrypt(self, buf): pos = common.ord(self.recv_buf[2]) + 2 out_buf += self.recv_buf[pos:length - 4] if not self.has_recv_header: - if len(out_buf) < 8: + if len(out_buf) < 12: self.raw_trans = True self.recv_buf = b'' + logging.info('auth_simple: too short') return b'E' - client_id = struct.unpack(' 60 * 3: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_simple: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) + return b'E' + elif self.server_info.data.insert(client_id, connection_id): self.has_recv_header = True - out_buf = out_buf[8:] + out_buf = out_buf[12:] self.client_id = client_id self.connection_id = connection_id else: self.raw_trans = True self.recv_buf = b'' + logging.info('auth_simple: auth fail, data %s' % (binascii.hexlify(out_buf),)) return b'E' self.recv_buf = self.recv_buf[length:] diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 006a3723..9734382f 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -68,7 +68,8 @@ def main(): obfs = config["obfs"] a_config = config.copy() ipv6_ok = False - logging.info("server start with password [%s] obfs [%s] method [%s]" % (password, obfs, a_config['method'])) + logging.info("server start with password [%s] method [%s] obfs [%s] obfs_param [%s]" % + (password, a_config['method'], obfs, a_config['obfs_param'])) if 'server_ipv6' in a_config: try: if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": @@ -77,7 +78,7 @@ def main(): a_config['password'] = password a_config['obfs'] = obfs a_config['server'] = a_config['server_ipv6'] - logging.info("starting server at %s:%d" % + logging.info("starting server at [%s]:%d" % (a_config['server'], int(port))) tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) From 83c8b50ea2bd43c22e82bfc1c9cfbf3234fac28d Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 23 Oct 2015 22:23:59 +0800 Subject: [PATCH 063/487] add startup time in auth_simple --- shadowsocks/obfsplugin/verify_simple.py | 3 ++- shadowsocks/tcprelay.py | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index c162c064..43f8c3e2 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -338,6 +338,7 @@ class obfs_auth_data(object): def __init__(self): self.sub_obfs = None self.client_id = {} + self.startup_time = int(time.time() - 30) & 0xffffffff def update(self, client_id, connection_id): if client_id in self.client_id: @@ -491,7 +492,7 @@ def server_post_decrypt(self, buf): client_id = struct.unpack(' 60 * 3: + if time_dif < 60 * -3 or time_dif > 60 * 3 or common.int32(utc_time - self.server_info.data.startup_time) < 0: self.raw_trans = True self.recv_buf = b'' logging.info('auth_simple: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 5d1f251f..f82762b7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -115,8 +115,6 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, config['method']) self._encrypt_correct = True self._obfs = obfs.obfs(config['obfs']) - if server.obfs_data is None: - server.obfs_data = self._obfs.init_data() server_info = obfs.server_info(server.obfs_data) server_info.host = config['server'] server_info.port = server._listen_port @@ -769,7 +767,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self.server_transfer_ul = 0 self.server_transfer_dl = 0 self.server_connections = 0 - self.obfs_data = None + self.obfs_data = obfs.obfs(config['obfs']).init_data() self._timeout = config['timeout'] self._timeouts = [] # a list for all the handlers From 59105f6e6469fc4d94e879fd4235202d237a8383 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 26 Oct 2015 14:43:17 +0800 Subject: [PATCH 064/487] support auth_simple --- shadowsocks/obfsplugin/verify_simple.py | 22 +++++++++++++++++++++- shadowsocks/tcprelay.py | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 43f8c3e2..22eb7879 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -338,7 +338,9 @@ class obfs_auth_data(object): def __init__(self): self.sub_obfs = None self.client_id = {} - self.startup_time = int(time.time() - 30) & 0xffffffff + self.startup_time = int(time.time() - 30) & 0xFFFFFFFF + self.local_client_id = b'' + self.connection_id = 0 def update(self, client_id, connection_id): if client_id in self.client_id: @@ -404,8 +406,26 @@ def pack_data(self, buf): data += struct.pack(' 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: ret += self.pack_data(buf[:self.unit_len]) buf = buf[self.unit_len:] diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f82762b7..578538a7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -263,8 +263,8 @@ def _write_to_sock(self, data, sock): if sock == self._local_sock and self._encrypt_correct: obfs_encode = self._obfs.server_encode(data) data = obfs_encode - l = len(data) - if l > 0: + if data: + l = len(data) s = sock.send(data) if s < l: data = data[s:] From 6ee803573538a79f73ede3adfca3a1c752bb1aa3 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 26 Oct 2015 14:48:54 +0800 Subject: [PATCH 065/487] merge pr from mengskysama --- shadowsocks/lru_cache.py | 2 +- shadowsocks/udprelay.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 401f19b5..e67fdffe 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -88,12 +88,12 @@ def sweep(self): self.close_callback(value) self._closed_values.add(value) for key in self._time_to_keys[least]: - self._last_visits.popleft() if key in self._store: if now - self._keys_to_last_time[key] > self.timeout: del self._store[key] del self._keys_to_last_time[key] c += 1 + self._last_visits.popleft() del self._time_to_keys[least] if c: self._closed_values.clear() diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index d3696d9e..b4fcae3e 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1293,6 +1293,7 @@ def handle_periodic(self): logging.info('closed UDP port %d', self._listen_port) before_sweep_size = len(self._sockets) self._cache.sweep() + self._dns_cache.sweep() if before_sweep_size != len(self._sockets): logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) self._client_fd_to_server_addr.sweep() From c8423e360dfc99c81fcaf0f7882385c17eb28b48 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 26 Oct 2015 16:37:27 +0800 Subject: [PATCH 066/487] DNS request ipv6 first --- server.py | 3 ++- shadowsocks/asyncdns.py | 44 ++++++++++++++++++++++++++++++++--------- shadowsocks/local.py | 9 ++++++++- shadowsocks/server.py | 9 ++++++++- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/server.py b/server.py index 2030397d..c93cee55 100644 --- a/server.py +++ b/server.py @@ -3,7 +3,8 @@ import sys import threading import os -os.chdir(os.path.split(os.path.realpath(__file__))[0]) + +os.chdir(os.path.dirname(os.path.realpath(__file__))) import server_pool import db_transfer diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index c5fc99d8..08ec5f0f 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -24,6 +24,13 @@ import re import logging +if __name__ == '__main__': + import sys + try: + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) + except: + sys.path.insert(0, os.path.join(sys.path[0], '../')) + from shadowsocks import common, lru_cache, eventloop, shell @@ -71,6 +78,17 @@ QTYPE_NS = 2 QCLASS_IN = 1 +def detect_ipv6_supprot(): + if 'has_ipv6' in dir(socket): + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + try: + s.connect(('ipv6.google.com', 0)) + return True + except: + pass + return False + +IPV6_CONNECTION_SUPPORT = detect_ipv6_supprot() def build_address(address): address = address.strip(b'.') @@ -338,17 +356,17 @@ def _handle_data(self, data): answer[2] == QCLASS_IN: ip = answer[0] break - if not ip and self._hostname_status.get(hostname, STATUS_IPV6) \ - == STATUS_IPV4: - self._hostname_status[hostname] = STATUS_IPV6 - self._send_req(hostname, QTYPE_AAAA) + if not ip and self._hostname_status.get(hostname, STATUS_IPV4) \ + == STATUS_IPV6: + self._hostname_status[hostname] = STATUS_IPV4 + self._send_req(hostname, QTYPE_A) else: if ip: self._cache[hostname] = ip self._call_callback(hostname, ip) - elif self._hostname_status.get(hostname, None) == STATUS_IPV6: + elif self._hostname_status.get(hostname, None) == STATUS_IPV4: for question in response.questions: - if question[1] == QTYPE_AAAA: + if question[1] == QTYPE_A: self._call_callback(hostname, None) break @@ -414,14 +432,21 @@ def resolve(self, hostname, callback): return arr = self._hostname_to_cb.get(hostname, None) if not arr: - self._hostname_status[hostname] = STATUS_IPV4 - self._send_req(hostname, QTYPE_A) + if IPV6_CONNECTION_SUPPORT: + self._hostname_status[hostname] = STATUS_IPV6 + self._send_req(hostname, QTYPE_AAAA) + else: + self._hostname_status[hostname] = STATUS_IPV4 + self._send_req(hostname, QTYPE_A) self._hostname_to_cb[hostname] = [callback] self._cb_to_hostname[callback] = hostname else: arr.append(callback) # TODO send again only if waited too long - self._send_req(hostname, QTYPE_A) + if IPV6_CONNECTION_SUPPORT: + self._send_req(hostname, QTYPE_AAAA) + else: + self._send_req(hostname, QTYPE_A) def close(self): if self._sock: @@ -479,3 +504,4 @@ def callback(result, error): if __name__ == '__main__': test() + diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 4255a2ee..79fd04e4 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -23,7 +23,14 @@ import logging import signal -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) +if __name__ == '__main__': + try: + os.chdir(os.path.dirname(os.path.realpath(__file__))) + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) + except: + os.chdir(os.path.dirname(sys.path[0])) + sys.path.insert(0, os.path.join(sys.path[0], '../')) + from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, asyncdns diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 9734382f..b6ef3c73 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -23,7 +23,14 @@ import logging import signal -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) +if __name__ == '__main__': + try: + os.chdir(os.path.dirname(os.path.realpath(__file__))) + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) + except: + os.chdir(os.path.dirname(sys.path[0])) + sys.path.insert(0, os.path.join(sys.path[0], '../')) + from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, \ asyncdns, manager From b31a8ea3f385abc4506a11ad9510642c3cd5a594 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 26 Oct 2015 16:53:16 +0800 Subject: [PATCH 067/487] auto get realpath --- server.py | 20 +++++++++++++++++++- shadowsocks/asyncdns.py | 8 ++++---- shadowsocks/local.py | 10 ++++------ shadowsocks/server.py | 10 ++++------ 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/server.py b/server.py index c93cee55..ac32a7c6 100644 --- a/server.py +++ b/server.py @@ -1,10 +1,28 @@ #!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright 2015 breakwall +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import time import sys import threading import os -os.chdir(os.path.dirname(os.path.realpath(__file__))) +if __name__ == '__main__': + import inspect + os.chdir(os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe())))) import server_pool import db_transfer diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 08ec5f0f..e64895c4 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -26,10 +26,10 @@ if __name__ == '__main__': import sys - try: - sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) - except: - sys.path.insert(0, os.path.join(sys.path[0], '../')) + import inspect + file_path = os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe()))) + os.chdir(file_path) + sys.path.insert(0, os.path.join(file_path, '../')) from shadowsocks import common, lru_cache, eventloop, shell diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 79fd04e4..c6df20ca 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -24,12 +24,10 @@ import signal if __name__ == '__main__': - try: - os.chdir(os.path.dirname(os.path.realpath(__file__))) - sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) - except: - os.chdir(os.path.dirname(sys.path[0])) - sys.path.insert(0, os.path.join(sys.path[0], '../')) + import inspect + file_path = os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe()))) + os.chdir(file_path) + sys.path.insert(0, os.path.join(file_path, '../')) from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, asyncdns diff --git a/shadowsocks/server.py b/shadowsocks/server.py index b6ef3c73..68d187fa 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -24,12 +24,10 @@ import signal if __name__ == '__main__': - try: - os.chdir(os.path.dirname(os.path.realpath(__file__))) - sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) - except: - os.chdir(os.path.dirname(sys.path[0])) - sys.path.insert(0, os.path.join(sys.path[0], '../')) + import inspect + file_path = os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe()))) + os.chdir(file_path) + sys.path.insert(0, os.path.join(file_path, '../')) from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, \ asyncdns, manager From 33d907cc08bc1037f86ebcf3b4d5f5c770e68e9e Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 26 Oct 2015 23:48:11 +0800 Subject: [PATCH 068/487] fix 'http_simple' & 'random_head' --- shadowsocks/manager.py | 6 +- shadowsocks/obfsplugin/http_simple.py | 90 +++++++++++++++++++++++---- shadowsocks/tcprelay.py | 10 ++- 3 files changed, 90 insertions(+), 16 deletions(-) diff --git a/shadowsocks/manager.py b/shadowsocks/manager.py index d5d9d12c..bfabd7fa 100644 --- a/shadowsocks/manager.py +++ b/shadowsocks/manager.py @@ -29,7 +29,7 @@ BUF_SIZE = 1506 -STAT_SEND_LIMIT = 100 +STAT_SEND_LIMIT = 50 class Manager(object): @@ -167,7 +167,9 @@ def send_data(data_dict): if i >= STAT_SEND_LIMIT: send_data(r) r.clear() - send_data(r) + i = 0 + if len(r) > 0 : + send_data(r) self._statistics.clear() def _send_control_data(self, data): diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 637015fd..9439d0dc 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -22,8 +22,10 @@ import hashlib import logging import binascii +import struct import base64 import datetime +import random from shadowsocks import common from shadowsocks.obfsplugin import plain @@ -66,14 +68,53 @@ def __init__(self, method): self.host = None self.port = 0 self.recv_buffer = b'' + self.user_agent = [b"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0", + b"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:40.0) Gecko/20100101 Firefox/44.0", + b"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.10 Chromium/27.0.1453.93 Chrome/27.0.1453.93 Safari/537.36", + b"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0", + b"Mozilla/5.0 (compatible; WOW64; MSIE 10.0; Windows NT 6.2)", + b"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + b"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; Trident/7.0; .NET4.0E; .NET4.0C)", + b"Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko", + b"Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/BuildID) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36", + b"Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3", + b"Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"] + + def encode_head(self, buf): + ret = b'' + for ch in buf: + ret += '%' + binascii.hexlify(ch) + return ret def client_encode(self, buf): - # TODO - return buf + if self.has_sent_header: + return buf + if len(buf) > 64: + headlen = random.randint(1, 64) + else: + headlen = len(buf) + headdata = buf[:headlen] + buf = buf[headlen:] + port = b'' + if self.server_info.port != 80: + port = b':' + common.to_bytes(str(self.server_info.port)) + http_head = b"GET /" + self.encode_head(headdata) + b" HTTP/1.1\r\n" + http_head += b"Host: " + (self.server_info.param or self.server_info.host) + port + b"\r\n" + http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" + http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\nDNT: 1\r\nConnection: keep-alive\r\n\r\n" + self.has_sent_header = True + return http_head + buf def client_decode(self, buf): - # TODO - return (buf, False) + if self.has_recv_header: + return (buf, False) + pos = buf.find(b'\r\n\r\n') + if pos >= 0: + self.has_recv_header = True + return (buf[pos + 4:], False) + else: + return (b'', False) def server_encode(self, buf): if self.has_sent_header: @@ -146,12 +187,19 @@ def __init__(self, method): self.recv_buffer = b'' def client_encode(self, buf): - # TODO - return buf + if self.has_sent_header: + return buf + #TODO def client_decode(self, buf): - # TODO - return (buf, False) + if self.has_recv_header: + return (buf, False) + pos = buf.find(b'\r\n\r\n') + if pos >= 0: + self.has_recv_header = True + return (buf[pos + 4:], False) + else: + return (b'', False) def server_encode(self, buf): if self.has_sent_header: @@ -204,9 +252,11 @@ def __init__(self, method): self.has_recv_header = False def client_encode(self, buf): + #TODO return buf def client_decode(self, buf): + #TODO # (buffer_to_recv, is_need_to_encode_and_send_back) return (buf, False) @@ -236,13 +286,31 @@ def __init__(self, method): self.method = method self.has_sent_header = False self.has_recv_header = False + self.raw_trans_sent = False + self.raw_trans_recv = False + self.send_buffer = b'' def client_encode(self, buf): - return buf + if self.raw_trans_sent: + return buf + self.send_buffer += buf + if not self.has_sent_header: + self.has_sent_header = True + data = os.urandom(common.ord(os.urandom(1)[0]) % 96 + 4) + crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff + return data + struct.pack(' Date: Tue, 27 Oct 2015 10:02:53 +0800 Subject: [PATCH 069/487] fix 'http2_simple' & 'tls_simple' --- shadowsocks/obfsplugin/http_simple.py | 43 ++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 9439d0dc..efd7f2a2 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -182,14 +182,28 @@ def __init__(self, method): self.method = method self.has_sent_header = False self.has_recv_header = False + self.raw_trans_sent = False self.host = None self.port = 0 self.recv_buffer = b'' def client_encode(self, buf): - if self.has_sent_header: + if self.raw_trans_sent: return buf - #TODO + self.send_buffer += buf + if not self.has_sent_header: + self.has_sent_header = True + http_head = b"GET / HTTP/1.1\r\n" + http_head += b"Host: " + (self.server_info.param or self.server_info.host) + port + b"\r\n" + http_head += b"Connection: Upgrade, HTTP2-Settings\r\nUpgrade: h2c\r\n" + http_head += b"HTTP2-Settings: " + base64.urlsafe_b64encode(buf) + b"\r\n" + return http_head + b"\r\n" + if self.has_recv_header: + ret = self.send_buffer + self.send_buffer = b'' + self.raw_trans_sent = True + return ret + return b'' def client_decode(self, buf): if self.has_recv_header: @@ -250,15 +264,30 @@ def __init__(self, method): self.method = method self.has_sent_header = False self.has_recv_header = False + self.raw_trans_sent = False def client_encode(self, buf): - #TODO - return buf + if self.raw_trans_sent: + return buf + self.send_buffer += buf + if not self.has_sent_header: + self.has_sent_header = True + data = b"\x03\x03" + os.urandom(32) + binascii.unhexlify(b"000016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") + data = b"\x01\x00" + struct.pack('>H', len(data)) + data + data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data + return data + if self.has_recv_header: + ret = self.send_buffer + self.send_buffer = b'' + self.raw_trans_sent = True + return ret + return b'' def client_decode(self, buf): - #TODO - # (buffer_to_recv, is_need_to_encode_and_send_back) - return (buf, False) + if self.has_recv_header: + return (buf, False) + self.has_recv_header = True + return (b'', True) def server_encode(self, buf): if self.has_sent_header: From 5137377580d85102be503673016582dfdcc4118b Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 27 Oct 2015 12:24:11 +0800 Subject: [PATCH 070/487] bump 2.6.12 --- CHANGES | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES b/CHANGES index 5cdbccdc..ef1acb26 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +2.6.12 2015-10-27 +- IPv6 first +- Fix mem leaks +- auth_simple plugin +- remove FORCE_NEW_PROTOCOL +- optimize code + 2.6.11 2015-10-20 - Obfs plugin - Obfs parameters From 8ee4cd513a92dac893266490ae9477b2810b839b Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 27 Oct 2015 19:13:38 +0800 Subject: [PATCH 071/487] a stupid bug --- shadowsocks/obfsplugin/verify_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 22eb7879..6b90a5a6 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -422,7 +422,7 @@ def auth_data(self): def client_pre_encrypt(self, buf): ret = b'' if not self.has_sent_header: - datalen = max(len(buf), common.ord(os.urandom(1)[0]) % 32 + 4) + datalen = min(len(buf), common.ord(os.urandom(1)[0]) % 32 + 4) ret += self.pack_data(self.auth_data() + buf[:datalen]) buf = buf[datalen:] self.has_sent_header = True From 011415f7109eb6c44b925aee9cbeff4719bd8de8 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 27 Oct 2015 21:10:45 +0800 Subject: [PATCH 072/487] auth package size not less than head size --- shadowsocks/obfsplugin/verify_simple.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 6b90a5a6..517de6d5 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -110,6 +110,17 @@ def server_decode(self, buf): return self.sub_obfs.server_decode(buf) return (buf, True, False) + def get_head_size(self, buf, def_value): + if len(buf) < 2: + return def_value + if ord(buf[0]) == 1: + return 7 + if ord(buf[0]) == 4: + return 19 + if ord(buf[0]) == 3: + return 4 + ord(buf[1]) + return def_value + class verify_simple(verify_base): def __init__(self, method): super(verify_simple, self).__init__(method) @@ -422,7 +433,8 @@ def auth_data(self): def client_pre_encrypt(self, buf): ret = b'' if not self.has_sent_header: - datalen = min(len(buf), common.ord(os.urandom(1)[0]) % 32 + 4) + head_size = self.get_head_size(buf, 30) + datalen = min(len(buf), random.randint(0, 31) + head_size) ret += self.pack_data(self.auth_data() + buf[:datalen]) buf = buf[datalen:] self.has_sent_header = True From e6d7a12afea37ec6f4fa3779a40693f6fef896f1 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 28 Oct 2015 22:28:24 +0800 Subject: [PATCH 073/487] fix bugs dns working in both v4 & v6 connecting problem in windows --- shadowsocks/asyncdns.py | 41 +++++++++++++++++++-------- shadowsocks/obfsplugin/http_simple.py | 3 ++ shadowsocks/tcprelay.py | 4 +-- shadowsocks/udprelay.py | 3 +- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index e64895c4..d958752e 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -83,9 +83,11 @@ def detect_ipv6_supprot(): s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) try: s.connect(('ipv6.google.com', 0)) + print('IPv6 support') return True except: pass + print('IPv6 not support') return False IPV6_CONNECTION_SUPPORT = detect_ipv6_supprot() @@ -356,19 +358,34 @@ def _handle_data(self, data): answer[2] == QCLASS_IN: ip = answer[0] break - if not ip and self._hostname_status.get(hostname, STATUS_IPV4) \ - == STATUS_IPV6: - self._hostname_status[hostname] = STATUS_IPV4 - self._send_req(hostname, QTYPE_A) + if IPV6_CONNECTION_SUPPORT: + if not ip and self._hostname_status.get(hostname, STATUS_IPV4) \ + == STATUS_IPV6: + self._hostname_status[hostname] = STATUS_IPV4 + self._send_req(hostname, QTYPE_A) + else: + if ip: + self._cache[hostname] = ip + self._call_callback(hostname, ip) + elif self._hostname_status.get(hostname, None) == STATUS_IPV4: + for question in response.questions: + if question[1] == QTYPE_A: + self._call_callback(hostname, None) + break else: - if ip: - self._cache[hostname] = ip - self._call_callback(hostname, ip) - elif self._hostname_status.get(hostname, None) == STATUS_IPV4: - for question in response.questions: - if question[1] == QTYPE_A: - self._call_callback(hostname, None) - break + if not ip and self._hostname_status.get(hostname, STATUS_IPV6) \ + == STATUS_IPV4: + self._hostname_status[hostname] = STATUS_IPV6 + self._send_req(hostname, QTYPE_AAAA) + else: + if ip: + self._cache[hostname] = ip + self._call_callback(hostname, ip) + elif self._hostname_status.get(hostname, None) == STATUS_IPV6: + for question in response.questions: + if question[1] == QTYPE_AAAA: + self._call_callback(hostname, None) + break def handle_event(self, sock, fd, event): if sock != self._sock: diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index efd7f2a2..8144aa80 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -192,6 +192,9 @@ def client_encode(self, buf): return buf self.send_buffer += buf if not self.has_sent_header: + port = b'' + if self.server_info.port != 80: + port = b':' + common.to_bytes(str(self.server_info.port)) self.has_sent_header = True http_head = b"GET / HTTP/1.1\r\n" http_head += b"Host: " + (self.server_info.param or self.server_info.host) + port + b"\r\n" diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index cce6df5b..e1ea9545 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -523,8 +523,8 @@ def _handle_dns_resolved(self, result, error): try: remote_sock.connect((remote_addr, remote_port)) except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == \ - errno.EINPROGRESS: + if eventloop.errno_from_exception(e) in (errno.EINPROGRESS, + errno.EWOULDBLOCK): pass # always goto here else: raise e diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index b4fcae3e..12676f7f 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -207,7 +207,7 @@ def insert(self, pack_id, data): def set_end(self, end_id): if end_id > self.end_id: eid = self.end_id - while eid < pack_id: + while eid < end_id: self.miss_queue.add(eid) eid += 1 self.end_id = end_id @@ -623,6 +623,7 @@ def sweep_timeout(self): for pid in missing: data += struct.pack(">H", pid) rsp_data = self._pack_post_data(CMD_SYN_STATUS, pack_id, data) + addr = self.get_local_address() self._write_to_sock(rsp_data, self._local_sock, addr) def handle_stream_sync_status(self, addr, cmd, request_id, pack_id, max_send_id, data): From c0d1d666a49f235cdd3d00b342358145216cac30 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 29 Oct 2015 11:00:01 +0800 Subject: [PATCH 074/487] add ipv6 config --- config.json | 1 + shadowsocks/local.py | 3 +++ shadowsocks/server.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/config.json b/config.json index d9a4c4be..7ca77c33 100644 --- a/config.json +++ b/config.json @@ -9,6 +9,7 @@ "method": "aes-256-cfb", "obfs": "http_simple_compatible", "obfs_param": "", + "ipv6": false, "fast_open": false, "workers": 1 } diff --git a/shadowsocks/local.py b/shadowsocks/local.py index c6df20ca..819e4423 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -43,6 +43,9 @@ def main(): config = shell.get_config(True) + if not config.get('ipv6', False): + asyncdns.IPV6_CONNECTION_SUPPORT = False + daemon.daemon_exec(config) try: diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 68d187fa..40615d36 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -54,6 +54,9 @@ def main(): else: config['port_password'][str(server_port)] = config['password'] + if not config.get('ipv6', False): + asyncdns.IPV6_CONNECTION_SUPPORT = False + if config.get('manager_address', 0): logging.info('entering manager mode') manager.run(config) From d9dc3ae0b5ccbfb874f56c9f32cbc1780c8cc215 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 29 Oct 2015 23:38:23 +0800 Subject: [PATCH 075/487] rename to "dns_ipv6" log wrong UDP header package --- config.json | 2 +- shadowsocks/local.py | 2 +- shadowsocks/server.py | 2 +- shadowsocks/udprelay.py | 32 ++++++++++++++++++++++++++------ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/config.json b/config.json index 7ca77c33..2fbf33e3 100644 --- a/config.json +++ b/config.json @@ -9,7 +9,7 @@ "method": "aes-256-cfb", "obfs": "http_simple_compatible", "obfs_param": "", - "ipv6": false, + "dns_ipv6": false, "fast_open": false, "workers": 1 } diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 819e4423..096283c1 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -43,7 +43,7 @@ def main(): config = shell.get_config(True) - if not config.get('ipv6', False): + if not config.get('dns_ipv6', False): asyncdns.IPV6_CONNECTION_SUPPORT = False daemon.daemon_exec(config) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 40615d36..235a3f2a 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -54,7 +54,7 @@ def main(): else: config['port_password'][str(server_port)] = config['password'] - if not config.get('ipv6', False): + if not config.get('dns_ipv6', False): asyncdns.IPV6_CONNECTION_SUPPORT = False if config.get('manager_address', 0): diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 12676f7f..5519466b 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -465,9 +465,11 @@ def _handle_dns_resolved(self, result, error): try: remote_sock.connect((remote_addr, remote_port)) except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == \ - errno.EINPROGRESS: - pass + if eventloop.errno_from_exception(e) in (errno.EINPROGRESS, + errno.EWOULDBLOCK): + pass # always goto here + else: + raise e self._loop.add(remote_sock, eventloop.POLL_ERR | eventloop.POLL_OUT, @@ -830,7 +832,10 @@ def destroy(self): if self._remote_sock: logging.debug('destroying remote') self._loop.remove(self._remote_sock) - del self._fd_to_handlers[self._remote_sock.fileno()] + try: + del self._fd_to_handlers[self._remote_sock.fileno()] + except Exception as e: + pass self._remote_sock.close() self._remote_sock = None if self._sendingqueue.empty(): @@ -845,7 +850,11 @@ def destroy_local(self): addr = self.get_local_address() self._write_to_sock(rsp_data, self._local_sock, addr) self._local_sock = None - del self._reqid_to_handlers[self._request_id] + try: + del self._reqid_to_handlers[self._request_id] + except Exception as e: + pass + self._server.remove_handler(self) def client_key(source_addr, server_af): @@ -964,9 +973,14 @@ def _pack_rsp_data(self, cmd, request_id, data): reqid_str = struct.pack(">H", request_id) return b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, data, _rand_data[:random.randint(0, len(_rand_data))], reqid_str]) + def _handel_protocol_error(self, client_address, ogn_data): + #raise Exception('can not parse header') + logging.warn("Protocol ERROR, UDP ogn data %s from %s:%d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1])) + def _handle_server(self): server = self._server_socket data, r_addr = server.recvfrom(BUF_SIZE) + ogn_data = data if not data: logging.debug('UDP handle_server: data is empty') if self._stat_callback: @@ -1057,8 +1071,14 @@ def _handle_server(self): logging.error(trace) return - header_result = parse_header(data) + try: + header_result = parse_header(data) + except: + self._handel_protocol_error(r_addr, ogn_data) + return + if header_result is None: + self._handel_protocol_error(r_addr, ogn_data) return connecttype, dest_addr, dest_port, header_length = header_result From c02fcbf746889dfed0b58bd30a0bd5e1e3b9f0c1 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 30 Oct 2015 10:22:04 +0800 Subject: [PATCH 076/487] refine time diff set to 5 min --- shadowsocks/obfsplugin/verify_simple.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 517de6d5..da63d52b 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -352,23 +352,24 @@ def __init__(self): self.startup_time = int(time.time() - 30) & 0xFFFFFFFF self.local_client_id = b'' self.connection_id = 0 + self.max_client = 16 # max active client count + self.max_buffer = max(self.max_client, 256) # max client id buffer size def update(self, client_id, connection_id): if client_id in self.client_id: self.client_id[client_id].update() def insert(self, client_id, connection_id): - max_client = 16 if client_id not in self.client_id or not self.client_id[client_id].enable: active = 0 for c_id in self.client_id: if self.client_id[c_id].is_active(): active += 1 - if active >= max_client: + if active >= self.max_client: logging.warn('auth_simple: max active clients exceeded') return False - if len(self.client_id) < max_client: + if len(self.client_id) < self.max_client: if client_id not in self.client_id: self.client_id[client_id] = client_queue(connection_id) else: @@ -378,7 +379,7 @@ def insert(self, client_id, connection_id): random.shuffle(keys) for c_id in keys: if not self.client_id[c_id].is_active() and self.client_id[c_id].enable: - if len(self.client_id) >= 256: + if len(self.client_id) >= self.max_buffer: del self.client_id[c_id] else: self.client_id[c_id].enable = False @@ -403,6 +404,7 @@ def __init__(self, method): self.has_recv_header = False self.client_id = 0 self.connection_id = 0 + self.max_time_dif = 60 * 5 # time dif (second) setting def init_data(self): return obfs_auth_data() @@ -524,7 +526,8 @@ def server_post_decrypt(self, buf): client_id = struct.unpack(' 60 * 3 or common.int32(utc_time - self.server_info.data.startup_time) < 0: + if time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ + or common.int32(utc_time - self.server_info.data.startup_time) < 0: self.raw_trans = True self.recv_buf = b'' logging.info('auth_simple: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) From fe4b35fb0a4418125633f1a82e03503c619b6884 Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 19:36:20 +0800 Subject: [PATCH 077/487] remove coverage run -a MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 最新版coverage的-a和-p参数不能同时用 --- tests/test.py | 2 +- tests/test_command.sh | 2 +- tests/test_daemon.sh | 2 +- tests/test_large_file.sh | 2 +- tests/test_udp_src.sh | 23 +++++++++++++++++++++++ 5 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 tests/test_udp_src.sh diff --git a/tests/test.py b/tests/test.py index 29b57d42..40834013 100755 --- a/tests/test.py +++ b/tests/test.py @@ -44,7 +44,7 @@ config = parser.parse_args() if config.with_coverage: - python = ['coverage', 'run', '-p', '-a'] + python = ['coverage', 'run', '-p'] client_args = python + ['shadowsocks/local.py', '-v'] server_args = python + ['shadowsocks/server.py', '-v'] diff --git a/tests/test_command.sh b/tests/test_command.sh index be05704f..696cb374 100755 --- a/tests/test_command.sh +++ b/tests/test_command.sh @@ -2,7 +2,7 @@ . tests/assert.sh -PYTHON="coverage run -a -p" +PYTHON="coverage run -p" LOCAL="$PYTHON shadowsocks/local.py" SERVER="$PYTHON shadowsocks/server.py" diff --git a/tests/test_daemon.sh b/tests/test_daemon.sh index 40f35ef0..7a192bdb 100755 --- a/tests/test_daemon.sh +++ b/tests/test_daemon.sh @@ -18,7 +18,7 @@ function run_test { for module in local server do -command="coverage run -p -a shadowsocks/$module.py" +command="coverage run -p shadowsocks/$module.py" mkdir -p tmp diff --git a/tests/test_large_file.sh b/tests/test_large_file.sh index 33bcb590..7a61caff 100755 --- a/tests/test_large_file.sh +++ b/tests/test_large_file.sh @@ -1,6 +1,6 @@ #!/bin/bash -PYTHON="coverage run -p -a" +PYTHON="coverage run -p" URL=http://127.0.0.1/file mkdir -p tmp diff --git a/tests/test_udp_src.sh b/tests/test_udp_src.sh new file mode 100644 index 00000000..6a778abc --- /dev/null +++ b/tests/test_udp_src.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +PYTHON="coverage run -p" + +mkdir -p tmp + +$PYTHON shadowsocks/local.py -c tests/aes.json -v & +LOCAL=$! + +$PYTHON shadowsocks/server.py -c tests/aes.json --forbidden-ip "" -v & +SERVER=$! + +sleep 3 + +python tests/test_udp_src.py +r=$? + +kill -s SIGINT $LOCAL +kill -s SIGINT $SERVER + +sleep 2 + +exit $r From 82b5980422c2786f4b08ab687bac8b4f6f491d6b Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 19:43:01 +0800 Subject: [PATCH 078/487] fix test files --- shadowsocks/common.py | 6 +++--- shadowsocks/encrypt_test.py | 8 ++++---- shadowsocks/manager.py | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 7f306ea8..d100d713 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -294,12 +294,12 @@ def test_inet_conv(): def test_parse_header(): assert parse_header(b'\x03\x0ewww.google.com\x00\x50') == \ - (3, b'www.google.com', 80, 18) + (0, b'www.google.com', 80, 18) assert parse_header(b'\x01\x08\x08\x08\x08\x00\x35') == \ - (1, b'8.8.8.8', 53, 7) + (0, b'8.8.8.8', 53, 7) assert parse_header((b'\x04$\x04h\x00@\x05\x08\x05\x00\x00\x00\x00\x00' b'\x00\x10\x11\x00\x50')) == \ - (4, b'2404:6800:4005:805::1011', 80, 19) + (0, b'2404:6800:4005:805::1011', 80, 19) def test_pack_header(): diff --git a/shadowsocks/encrypt_test.py b/shadowsocks/encrypt_test.py index 68228e18..0121d633 100644 --- a/shadowsocks/encrypt_test.py +++ b/shadowsocks/encrypt_test.py @@ -7,10 +7,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) -from crypto import rc4_md5 -from crypto import openssl -from crypto import sodium -from crypto import table +from shadowsocks.crypto import rc4_md5 +from shadowsocks.crypto import openssl +from shadowsocks.crypto import sodium +from shadowsocks.crypto import table def main(): print("\n""rc4_md5") diff --git a/shadowsocks/manager.py b/shadowsocks/manager.py index bfabd7fa..90092536 100644 --- a/shadowsocks/manager.py +++ b/shadowsocks/manager.py @@ -207,7 +207,9 @@ def test(): eventloop.TIMEOUT_PRECISION = 1 def run_server(): - config = { + config = shell.get_config(True) + config = config.copy() + a_config = { 'server': '127.0.0.1', 'local_port': 1081, 'port_password': { @@ -220,7 +222,8 @@ def run_server(): 'fast_open': False, 'verbose': 2 } - manager = Manager(config) + a_config.update(config) + manager = Manager(a_config) enc.append(manager) manager.run() From 72001556b52a09ba939f1701bfccfd00e1ea40a7 Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 19:46:18 +0800 Subject: [PATCH 079/487] remove m2.py --- shadowsocks/crypto/m2.py | 117 --------------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 shadowsocks/crypto/m2.py diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py deleted file mode 100644 index 4c7e1480..00000000 --- a/shadowsocks/crypto/m2.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2014 clowwindy -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from __future__ import absolute_import, division, print_function, \ - with_statement - -import sys -import logging - -__all__ = ['ciphers'] - -has_m2 = True -try: - __import__('M2Crypto') -except ImportError: - has_m2 = False - - -def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, - padding=1): - - import M2Crypto.EVP - return M2Crypto.EVP.Cipher(alg.replace('-', '_'), key, iv, op, - key_as_bytes=0, d='md5', salt=None, i=1, - padding=1) - - -def err(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): - logging.error(('M2Crypto is required to use %s, please run' - ' `apt-get install python-m2crypto`') % alg) - sys.exit(1) - - -if has_m2: - ciphers = { - b'aes-128-cfb': (16, 16, create_cipher), - b'aes-192-cfb': (24, 16, create_cipher), - b'aes-256-cfb': (32, 16, create_cipher), - b'bf-cfb': (16, 8, create_cipher), - b'camellia-128-cfb': (16, 16, create_cipher), - b'camellia-192-cfb': (24, 16, create_cipher), - b'camellia-256-cfb': (32, 16, create_cipher), - b'cast5-cfb': (16, 8, create_cipher), - b'des-cfb': (8, 8, create_cipher), - b'idea-cfb': (16, 8, create_cipher), - b'rc2-cfb': (16, 8, create_cipher), - b'rc4': (16, 0, create_cipher), - b'seed-cfb': (16, 16, create_cipher), - } -else: - ciphers = {} - - -def run_method(method): - from shadowsocks.crypto import util - - cipher = create_cipher(method, b'k' * 32, b'i' * 16, 1) - decipher = create_cipher(method, b'k' * 32, b'i' * 16, 0) - - util.run_cipher(cipher, decipher) - - -def check_env(): - # skip this test on pypy and Python 3 - try: - import __pypy__ - del __pypy__ - from nose.plugins.skip import SkipTest - raise SkipTest - except ImportError: - pass - if bytes != str: - from nose.plugins.skip import SkipTest - raise SkipTest - - -def test_aes_128_cfb(): - check_env() - run_method(b'aes-128-cfb') - - -def test_aes_256_cfb(): - check_env() - run_method(b'aes-256-cfb') - - -def test_bf_cfb(): - check_env() - run_method(b'bf-cfb') - - -def test_rc4(): - check_env() - run_method(b'rc4') - - -if __name__ == '__main__': - test_aes_128_cfb() From fcc7a4cf9bc43af75fe91585d9129d7d03ecd536 Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 20:35:01 +0800 Subject: [PATCH 080/487] remove os.chdir(file_path) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit os.chdir(file_path) 会导致测试代码无法找到引用的包，暂时先移除。 --- shadowsocks/asyncdns.py | 1 - shadowsocks/local.py | 1 - shadowsocks/server.py | 1 - 3 files changed, 3 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index d958752e..dd329024 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -28,7 +28,6 @@ import sys import inspect file_path = os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe()))) - os.chdir(file_path) sys.path.insert(0, os.path.join(file_path, '../')) from shadowsocks import common, lru_cache, eventloop, shell diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 096283c1..68edd6fc 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -26,7 +26,6 @@ if __name__ == '__main__': import inspect file_path = os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe()))) - os.chdir(file_path) sys.path.insert(0, os.path.join(file_path, '../')) from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, asyncdns diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 235a3f2a..14ff930e 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -26,7 +26,6 @@ if __name__ == '__main__': import inspect file_path = os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe()))) - os.chdir(file_path) sys.path.insert(0, os.path.join(file_path, '../')) from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, \ From 7d2b66766c84fd3ac555ec33103d91e97510e05c Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 20:36:44 +0800 Subject: [PATCH 081/487] add cymysql --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f29cb960..3e20958f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ before_install: - sudo dd if=/dev/urandom of=/usr/share/nginx/www/file bs=1M count=10 - sudo sh -c "echo '127.0.0.1 localhost' > /etc/hosts" - sudo service nginx restart - - pip install pep8 pyflakes nose coverage + - pip install pep8 pyflakes nose coverage cymysql - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh - sudo tests/setup_tc.sh From 208f1462c395717f9d951ed479ecaadd4952c6dc Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 20:41:52 +0800 Subject: [PATCH 082/487] fix manger test again --- shadowsocks/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/manager.py b/shadowsocks/manager.py index 90092536..57b95b70 100644 --- a/shadowsocks/manager.py +++ b/shadowsocks/manager.py @@ -222,8 +222,8 @@ def run_server(): 'fast_open': False, 'verbose': 2 } - a_config.update(config) - manager = Manager(a_config) + config.update(a_config) + manager = Manager(config) enc.append(manager) manager.run() From 60e86c27c4c517bfcff4132c6a269a165160d984 Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 21:38:27 +0800 Subject: [PATCH 083/487] add setup.py and remove __init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根目录下的__init__.py会导致测试代码无法找到引用的包。 --- __init__.py | 1 - setup.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) delete mode 100644 __init__.py create mode 100644 setup.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 013e4b7e..00000000 --- a/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/python diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..689dd736 --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +import codecs +from setuptools import setup + + +with codecs.open('README.rst', encoding='utf-8') as f: + long_description = f.read() + +setup( + name="shadowsocks", + version="2.6.12", + license='http://www.apache.org/licenses/LICENSE-2.0', + description="A fast tunnel proxy that help you get through firewalls", + author='clowwindy', + author_email='clowwindy42@gmail.com', + url='https://github.com/shadowsocks/shadowsocks', + packages=['shadowsocks', 'shadowsocks.crypto'], + package_data={ + 'shadowsocks': ['README.rst', 'LICENSE'] + }, + install_requires=[], + entry_points=""" + [console_scripts] + sslocal = shadowsocks.local:main + ssserver = shadowsocks.server:main + """, + classifiers=[ + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Topic :: Internet :: Proxy Servers', + ], + long_description=long_description, +) From 52391b9214f3e73cec7053e0d838f659c0ad28b7 Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 22:00:25 +0800 Subject: [PATCH 084/487] add test_udp_src --- tests/test_udp_src.py | 83 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/test_udp_src.py diff --git a/tests/test_udp_src.py b/tests/test_udp_src.py new file mode 100644 index 00000000..e8fa5057 --- /dev/null +++ b/tests/test_udp_src.py @@ -0,0 +1,83 @@ +#!/usr/bin/python + +import socket +import socks + + +SERVER_IP = '127.0.0.1' +SERVER_PORT = 1081 + + +if __name__ == '__main__': + # Test 1: same source port IPv4 + sock_out = socks.socksocket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + sock_out.set_proxy(socks.SOCKS5, SERVER_IP, SERVER_PORT) + sock_out.bind(('127.0.0.1', 9000)) + + sock_in1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + sock_in2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + + sock_in1.bind(('127.0.0.1', 9001)) + sock_in2.bind(('127.0.0.1', 9002)) + + sock_out.sendto(b'data', ('127.0.0.1', 9001)) + result1 = sock_in1.recvfrom(8) + + sock_out.sendto(b'data', ('127.0.0.1', 9002)) + result2 = sock_in2.recvfrom(8) + + sock_out.close() + sock_in1.close() + sock_in2.close() + + # make sure they're from the same source port + assert result1 == result2 + + # Test 2: same source port IPv6 + # try again from the same port but IPv6 + sock_out = socks.socksocket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + sock_out.set_proxy(socks.SOCKS5, SERVER_IP, SERVER_PORT) + sock_out.bind(('127.0.0.1', 9000)) + + sock_in1 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM, + socket.SOL_UDP) + sock_in2 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM, + socket.SOL_UDP) + + sock_in1.bind(('::1', 9001)) + sock_in2.bind(('::1', 9002)) + + sock_out.sendto(b'data', ('::1', 9001)) + result1 = sock_in1.recvfrom(8) + + sock_out.sendto(b'data', ('::1', 9002)) + result2 = sock_in2.recvfrom(8) + + sock_out.close() + sock_in1.close() + sock_in2.close() + + # make sure they're from the same source port + assert result1 == result2 + + # Test 3: different source ports IPv6 + sock_out = socks.socksocket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + sock_out.set_proxy(socks.SOCKS5, SERVER_IP, SERVER_PORT) + sock_out.bind(('127.0.0.1', 9003)) + + sock_in1 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM, + socket.SOL_UDP) + sock_in1.bind(('::1', 9001)) + sock_out.sendto(b'data', ('::1', 9001)) + result3 = sock_in1.recvfrom(8) + + # make sure they're from different source ports + assert result1 != result3 + + sock_out.close() + sock_in1.close() From a96e2cbb631668e4e7f49a32bfe3f5e48570904c Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 22:26:03 +0800 Subject: [PATCH 085/487] fix READEME and travis url --- .travis.yml | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e20958f..014fa071 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ before_install: - sudo dd if=/dev/urandom of=/usr/share/nginx/www/file bs=1M count=10 - sudo sh -c "echo '127.0.0.1 localhost' > /etc/hosts" - sudo service nginx restart - - pip install pep8 pyflakes nose coverage cymysql + - pip install pep8 pyflakes nose coverage PySocks cymysql - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh - sudo tests/setup_tc.sh diff --git a/README.md b/README.md index 76d759a2..9c38e3e4 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Bugs and Issues [Android]: https://github.com/shadowsocks/shadowsocks-android -[Build Status]: https://img.shields.io/travis/shadowsocks/shadowsocks/master.svg?style=flat +[Build Status]: https://travis-ci.org/falseen/shadowsocks.svg?branch=manyuser-travis [Configuration]: https://github.com/shadowsocks/shadowsocks/wiki/Configuration-via-Config-File [Coverage Status]: https://jenkins.shadowvpn.org/result/shadowsocks [Coverage]: https://jenkins.shadowvpn.org/job/Shadowsocks/ws/PYENV/py34/label/linux/htmlcov/index.html @@ -100,7 +100,7 @@ Bugs and Issues [OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help [PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat -[Travis CI]: https://travis-ci.org/shadowsocks/shadowsocks +[Travis CI]: https://travis-ci.org/falseen/shadowsocks [Troubleshooting]: https://github.com/shadowsocks/shadowsocks/wiki/Troubleshooting [Wiki]: https://github.com/shadowsocks/shadowsocks/wiki [Windows]: https://github.com/shadowsocks/shadowsocks-csharp From 0f7a0c562845339cef3a7372b0781977c24fae1e Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 23:02:01 +0800 Subject: [PATCH 086/487] fix chmod --- tests/assert.sh | 0 tests/test_udp_src.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/assert.sh mode change 100644 => 100755 tests/test_udp_src.sh diff --git a/tests/assert.sh b/tests/assert.sh old mode 100644 new mode 100755 diff --git a/tests/test_udp_src.sh b/tests/test_udp_src.sh old mode 100644 new mode 100755 From 032027c3ff106c431464dcb6f612f060fb15bdad Mon Sep 17 00:00:00 2001 From: Falseen Date: Sun, 1 Nov 2015 23:56:37 +0800 Subject: [PATCH 087/487] fix test_command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 由于ssr在根目录中添加了config.json文件，导致相关的测试代码失效。所以暂时先移除。 --- tests/test_command.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/test_command.sh b/tests/test_command.sh index 696cb374..a1a777b0 100755 --- a/tests/test_command.sh +++ b/tests/test_command.sh @@ -9,11 +9,6 @@ SERVER="$PYTHON shadowsocks/server.py" assert "$LOCAL --version 2>&1 | grep Shadowsocks | awk -F\" \" '{print \$1}'" "Shadowsocks" assert "$SERVER --version 2>&1 | grep Shadowsocks | awk -F\" \" '{print \$1}'" "Shadowsocks" -assert "$LOCAL 2>&1 | grep ERROR" "ERROR: config not specified" -assert "$LOCAL 2>&1 | grep usage | cut -d: -f1" "usage" - -assert "$SERVER 2>&1 | grep ERROR" "ERROR: config not specified" -assert "$SERVER 2>&1 | grep usage | cut -d: -f1" "usage" assert "$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: server set to listen on 127.0.0.1:8388, are you sure?" $LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop @@ -30,14 +25,6 @@ $LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d sto assert "$LOCAL 2>&1 -m rc4-md5 -k mypassword -s 0.0.0.0 -p 8388 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" " DON'T USE DEFAULT PASSWORD! Please change it in your config.json!" $LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop -assert "$LOCAL 2>&1 -m rc4-md5 -p 8388 -k testrc4 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": server addr not specified" -$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop - -assert "$LOCAL 2>&1 -m rc4-md5 -p 8388 -s 0.0.0.0 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" " password not specified" -$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop - -assert "$SERVER 2>&1 -m rc4-md5 -p 8388 -s 0.0.0.0 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" " password or port_password not specified" -$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$SERVER 2>&1 --forbidden-ip 127.0.0.1/4a -m rc4-md5 -k 12345 -p 8388 -s 0.0.0.0 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": Not a valid CIDR notation: 127.0.0.1/4a" $LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop From cb17c7b3b510570d565176a501e5adf106aac47c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 2 Nov 2015 15:13:54 +0800 Subject: [PATCH 088/487] let setting & code simple add "protocol" in config.json --- config.json | 1 + shadowsocks/obfs.py | 4 ++++ shadowsocks/obfsplugin/plain.py | 4 ++++ shadowsocks/obfsplugin/verify_simple.py | 31 +------------------------ shadowsocks/shell.py | 1 + shadowsocks/tcprelay.py | 27 ++++++++++++++++----- 6 files changed, 32 insertions(+), 36 deletions(-) diff --git a/config.json b/config.json index 2fbf33e3..97cd259e 100644 --- a/config.json +++ b/config.json @@ -7,6 +7,7 @@ "password": "m", "timeout": 300, "method": "aes-256-cfb", + "protocol": "origin", "obfs": "http_simple_compatible", "obfs_param": "", "dns_ipv6": false, diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 92de548f..1752a56e 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -83,3 +83,7 @@ def server_decode(self, buf): def server_post_decrypt(self, buf): return self.obfs.server_post_decrypt(buf) + def dispose(self): + self.obfs.dispose() + del self.obfs + diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index cc602fae..5450e7a2 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -27,6 +27,7 @@ def create_obfs(method): obfs_map = { 'plain': (create_obfs,), + 'origin': (create_obfs,), } class plain(object): @@ -66,3 +67,6 @@ def server_decode(self, buf): def server_post_decrypt(self, buf): return buf + def dispose(self): + pass + diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index da63d52b..8945d964 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -57,57 +57,29 @@ def match_begin(str1, str2): class obfs_verify_data(object): def __init__(self): - self.sub_obfs = None + pass class verify_base(plain.plain): def __init__(self, method): super(verify_base, self).__init__(method) self.method = method - self.sub_obfs = None def init_data(self): return obfs_verify_data() def set_server_info(self, server_info): - try: - if server_info.param: - sub_param = '' - param_list = server_info.param.split(',', 1) - if len(param_list) > 1: - self.sub_obfs = shadowsocks.obfs.obfs(param_list[0]) - sub_param = param_list[1] - else: - self.sub_obfs = shadowsocks.obfs.obfs(server_info.param) - if server_info.data.sub_obfs is None: - server_info.data.sub_obfs = self.sub_obfs.init_data() - _server_info = shadowsocks.obfs.server_info(server_info.data.sub_obfs) - _server_info.host = server_info.host - _server_info.port = server_info.port - _server_info.tcp_mss = server_info.tcp_mss - _server_info.param = sub_param - self.sub_obfs.set_server_info(_server_info) - except Exception as e: - shadowsocks.shell.print_exception(e) self.server_info = server_info def client_encode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.client_encode(buf) return buf def client_decode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.client_decode(buf) return (buf, False) def server_encode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.server_encode(buf) return buf def server_decode(self, buf): - if self.sub_obfs is not None: - return self.sub_obfs.server_decode(buf) return (buf, True, False) def get_head_size(self, buf, def_value): @@ -347,7 +319,6 @@ def insert(self, connection_id): class obfs_auth_data(object): def __init__(self): - self.sub_obfs = None self.client_id = {} self.startup_time = int(time.time() - 30) & 0xFFFFFFFF self.local_client_id = b'' diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 66e38bb5..38d2432b 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -218,6 +218,7 @@ def get_config(is_local): config['password'] = to_bytes(config.get('password', b'')) config['method'] = to_str(config.get('method', 'aes-256-cfb')) + config['protocol'] = to_str(config.get('protocol', 'origin')) config['obfs'] = to_str(config.get('obfs', 'plain')) config['obfs_param'] = to_str(config.get('obfs_param', '')) config['port_password'] = config.get('port_password', None) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e1ea9545..395ecbf9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -122,6 +122,14 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.param = config['obfs_param'] self._obfs.set_server_info(server_info) + self._protocol = obfs.obfs(config['protocol']) + server_info = obfs.server_info(server.protocol_data) + server_info.host = config['server'] + server_info.port = server._listen_port + server_info.tcp_mss = 1440 + server_info.param = '' + self._protocol.set_server_info(server_info) + self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] @@ -330,7 +338,7 @@ def _handel_protocol_error(self, client_address, ogn_data): def _handle_stage_connecting(self, data): if self._is_local: - data = self._obfs.client_pre_encrypt(data) + data = self._protocol.client_pre_encrypt(data) data = self._encryptor.encrypt(data) data = self._obfs.client_encode(data) if data: @@ -428,7 +436,7 @@ def _handle_stage_addr(self, ogn_data, data): data = b'\x88' + struct.pack('>H', total_len) + chr(rnd_len) + (b' ' * (rnd_len - 1)) + data crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff data += struct.pack(' Date: Mon, 2 Nov 2015 16:48:24 +0800 Subject: [PATCH 089/487] multiple-port setting --- shadowsocks/server.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 235a3f2a..651f209c 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -68,23 +68,32 @@ def main(): port_password = config['port_password'] del config['port_password'] for port, password_obfs in port_password.items(): + protocol = 'origin' + obfs_param = '' if type(password_obfs) == list: password = password_obfs[0] obfs = password_obfs[1] + elif type(password_obfs) == dict: + password = password_obfs.get('password', 'm') + protocol = password_obfs.get('protocol', 'origin') + obfs = password_obfs.get('obfs', 'plain') + obfs_param = password_obfs.get('obfs_param', '') else: password = password_obfs obfs = config["obfs"] a_config = config.copy() ipv6_ok = False - logging.info("server start with password [%s] method [%s] obfs [%s] obfs_param [%s]" % - (password, a_config['method'], obfs, a_config['obfs_param'])) + logging.info("server start with protocol[%s] password [%s] method [%s] obfs [%s] obfs_param [%s]" % + (protocol, password, a_config['method'], obfs, obfs_param)) if 'server_ipv6' in a_config: try: if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": a_config['server_ipv6'] = a_config['server_ipv6'][1:-1] a_config['server_port'] = int(port) a_config['password'] = password + a_config['protocol'] = protocol a_config['obfs'] = obfs + a_config['obfs_param'] = obfs_param a_config['server'] = a_config['server_ipv6'] logging.info("starting server at [%s]:%d" % (a_config['server'], int(port))) @@ -99,7 +108,9 @@ def main(): a_config = config.copy() a_config['server_port'] = int(port) a_config['password'] = password + a_config['protocol'] = protocol a_config['obfs'] = obfs + a_config['obfs_param'] = obfs_param logging.info("starting server at %s:%d" % (a_config['server'], int(port))) tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) From bc419c4dc2bbc3a285eca36067f7cc9ea7628715 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 2 Nov 2015 17:00:35 +0800 Subject: [PATCH 090/487] bump 2.6.13 --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index ef1acb26..a58348a4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.6.13 2015-11-02 +- add protocol setting + 2.6.12 2015-10-27 - IPv6 first - Fix mem leaks From b606e19c0bd284b388a148d87a647665aca97df8 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 2 Nov 2015 19:42:45 +0800 Subject: [PATCH 091/487] fix default protocol setting --- shadowsocks/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 651f209c..4c19474f 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -68,8 +68,8 @@ def main(): port_password = config['port_password'] del config['port_password'] for port, password_obfs in port_password.items(): - protocol = 'origin' - obfs_param = '' + protocol = config.get("protocol", 'origin') + obfs_param = config.get("obfs_param", '') if type(password_obfs) == list: password = password_obfs[0] obfs = password_obfs[1] From 5e46aa0b47a42418981b59ccb4af622ebef3b37f Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 3 Nov 2015 18:17:16 +0800 Subject: [PATCH 092/487] check encryptor object --- shadowsocks/tcprelay.py | 80 +++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 395ecbf9..028e33d2 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -268,7 +268,7 @@ def _write_to_sock(self, data, sock): if self._is_local: pass else: - if sock == self._local_sock and self._encrypt_correct: + if sock == self._local_sock and self._encrypt_correct and (self._obfs is not None): obfs_encode = self._obfs.server_encode(data) data = obfs_encode if data: @@ -338,9 +338,10 @@ def _handel_protocol_error(self, client_address, ogn_data): def _handle_stage_connecting(self, data): if self._is_local: - data = self._protocol.client_pre_encrypt(data) - data = self._encryptor.encrypt(data) - data = self._obfs.client_encode(data) + if self._encryptor is not None: + data = self._protocol.client_pre_encrypt(data) + data = self._encryptor.encrypt(data) + data = self._obfs.client_encode(data) if data: self._data_to_write_to_remote.append(data) if self._is_local and not self._fastopen_connected and \ @@ -436,9 +437,10 @@ def _handle_stage_addr(self, ogn_data, data): data = b'\x88' + struct.pack('>H', total_len) + chr(rnd_len) + (b' ' * (rnd_len - 1)) + data crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff data += struct.pack(' Date: Thu, 5 Nov 2015 19:25:19 +0800 Subject: [PATCH 093/487] fix http_simple a bug of urlencode length --- shadowsocks/obfsplugin/http_simple.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 8144aa80..ad782ee4 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -133,10 +133,14 @@ def get_data_from_http_header(self, buf): hex_items = lines[0].split(b'%') if hex_items and len(hex_items) > 1: for index in range(1, len(hex_items)): - if len(hex_items[index]) != 2: + if len(hex_items[index]) < 2: + ret_buf += binascii.unhexlify('0' + hex_items[index]) + break + elif len(hex_items[index]) > 2: ret_buf += binascii.unhexlify(hex_items[index][:2]) break - ret_buf += binascii.unhexlify(hex_items[index]) + else: + ret_buf += binascii.unhexlify(hex_items[index]) return ret_buf return b'' From 04133d806c583e672945b0ae0ae1420877f2e322 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 5 Nov 2015 20:18:12 +0800 Subject: [PATCH 094/487] set default timeout 120 for many connections --- config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json b/config.json index 97cd259e..3be6e567 100644 --- a/config.json +++ b/config.json @@ -5,7 +5,7 @@ "local_address": "127.0.0.1", "local_port": 1080, "password": "m", - "timeout": 300, + "timeout": 120, "method": "aes-256-cfb", "protocol": "origin", "obfs": "http_simple_compatible", From 877f0470c50d7a9b42e1f7caf3e572d1a55bd717 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 8 Nov 2015 23:35:04 +0800 Subject: [PATCH 095/487] http_simple: fix no response in some case verify_simple: check package size minimum 7 bytes --- shadowsocks/obfsplugin/http_simple.py | 5 +++-- shadowsocks/obfsplugin/verify_simple.py | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index ad782ee4..1edf1218 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -171,9 +171,10 @@ def server_decode(self, buf): return (b'', True, False) datas = buf.split(b'\r\n\r\n', 1) - if datas and len(datas) > 1: + if datas: ret_buf = self.get_data_from_http_header(buf) - ret_buf += datas[1] + if len(datas) > 1: + ret_buf += datas[1] if len(ret_buf) >= 15: self.has_recv_header = True return (ret_buf, True, False) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 8945d964..03c73b22 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -126,7 +126,7 @@ def client_post_decrypt(self, buf): out_buf = b'' while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192: + if length >= 8192 or length < 7: self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: @@ -167,7 +167,7 @@ def server_post_decrypt(self, buf): out_buf = b'' while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192: + if length >= 8192 or length < 7: self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: @@ -223,7 +223,7 @@ def client_post_decrypt(self, buf): out_buf = b'' while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 32768: + if length >= 32768 or length < 6: self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: @@ -255,7 +255,7 @@ def server_post_decrypt(self, buf): out_buf = b'' while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 32768: + if length >= 32768 or length < 6: self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: @@ -424,7 +424,7 @@ def client_post_decrypt(self, buf): out_buf = b'' while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192: + if length >= 8192 or length < 7: self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: @@ -465,7 +465,7 @@ def server_post_decrypt(self, buf): out_buf = b'' while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192: + if length >= 8192 or length < 7: self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: From b4d1d6b5f4ee7904b97f898eb89d643d9942e5ce Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 11 Nov 2015 10:23:21 +0800 Subject: [PATCH 096/487] nginx http response --- shadowsocks/obfsplugin/http_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 1edf1218..5b2e4801 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -120,9 +120,9 @@ def server_encode(self, buf): if self.has_sent_header: return buf - header = b'HTTP/1.1 200 OK\r\nServer: openresty\r\nDate: ' + header = b'HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nContent-Encoding: gzip\r\nContent-Type: text/html\r\nDate: ' header += to_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT')) - header += b'\r\nContent-Type: text/plain; charset=utf-8\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\nKeep-Alive: timeout=20\r\nVary: Accept-Encoding\r\nContent-Encoding: gzip\r\n\r\n' + header += b'\r\nServer: nginx\r\nTransfer-Encoding: chunked\r\nVary: Accept-Encoding\r\n\r\n' self.has_sent_header = True return header + buf From f45d57950be45dbaccd291ec9e8b60953019e71c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 11 Nov 2015 22:45:09 +0800 Subject: [PATCH 097/487] default buffer setting --- shadowsocks/tcprelay.py | 4 ---- shadowsocks/udprelay.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 028e33d2..e8ae7754 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -480,10 +480,6 @@ def _create_remote_socket(self, ip, port): remote_sock_v6 = socket.socket(af, socktype, proto) self._remote_sock_v6 = remote_sock_v6 self._fd_to_handlers[remote_sock_v6.fileno()] = self - remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) - remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) - remote_sock_v6.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) - remote_sock_v6.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) remote_sock.setblocking(False) if self._remote_udp: diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 5519466b..c6a69b70 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -914,8 +914,6 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): server_socket = socket.socket(af, socktype, proto) server_socket.bind((self._listen_addr, self._listen_port)) server_socket.setblocking(False) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024 * 32) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024 * 32) self._server_socket = server_socket self._stat_callback = stat_callback From b778a60502ec6bdd8996f9c78e9b55163d568a59 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 12 Nov 2015 17:48:50 +0800 Subject: [PATCH 098/487] detect wrong http request disconnect wrong request by default --- shadowsocks/obfsplugin/http_simple.py | 2 ++ shadowsocks/tcprelay.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 5b2e4801..4dc8d494 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -178,6 +178,8 @@ def server_decode(self, buf): if len(ret_buf) >= 15: self.has_recv_header = True return (ret_buf, True, False) + if len(ret_buf) == 0: + return self.not_match_return(buf) return (b'', True, False) else: return (b'', True, False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e8ae7754..355b2111 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -130,6 +130,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.param = '' self._protocol.set_server_info(server_info) + self._redir_list = config.get('redirect', ["0.0.0.0:0"]) + self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] @@ -312,8 +314,7 @@ def _write_to_sock(self, data, sock): return True def _get_redirect_host(self, client_address, ogn_data): - # test - host_list = [(b"www.bing.com", 80), (b"www.microsoft.com", 80), (b"cloudfront.com", 80), (b"cloudflare.com", 80), (b"1.2.3.4", 1000), (b"0.0.0.0", 0)] + host_list = self._redir_list or ["0.0.0.0:0"] hash_code = binascii.crc32(ogn_data) addrs = socket.getaddrinfo(client_address[0], client_address[1], 0, socket.SOCK_STREAM, socket.SOL_TCP) af, socktype, proto, canonname, sa = addrs[0] @@ -324,7 +325,14 @@ def _get_redirect_host(self, client_address, ogn_data): addr = struct.unpack('>I', address_bytes)[0] else: addr = 0 - return host_list[((hash_code & 0xffffffff) + addr + 3) % len(host_list)] + host_post = common.to_str(host_list[((hash_code & 0xffffffff) + addr) % len(host_list)]) + items = host_post.rsplit(':', 1) + if len(items) > 1: + try: + return (items[0], int(items[1])) + except: + pass + return (host_post, 80) def _handel_protocol_error(self, client_address, ogn_data): #raise Exception('can not parse header') @@ -332,7 +340,7 @@ def _handel_protocol_error(self, client_address, ogn_data): self._encrypt_correct = False #create redirect or disconnect by hash code host, port = self._get_redirect_host(client_address, ogn_data) - data = b"\x03" + common.chr(len(host)) + host + struct.pack('>H', port) + data = b"\x03" + common.chr(len(host)) + common.to_bytes(host) + struct.pack('>H', port) logging.warn("TCP data redir %s:%d %s" % (host, port, binascii.hexlify(data))) return data + ogn_data From 2a03747e76caa4a1e6f2e51ee7f6446ca2bfac1c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 12 Nov 2015 17:59:21 +0800 Subject: [PATCH 099/487] pack header in uri --- shadowsocks/obfsplugin/http_simple.py | 5 +++-- shadowsocks/obfsplugin/plain.py | 14 ++++++++++++++ shadowsocks/obfsplugin/verify_simple.py | 11 ----------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 4dc8d494..926ba857 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -90,8 +90,9 @@ def encode_head(self, buf): def client_encode(self, buf): if self.has_sent_header: return buf - if len(buf) > 64: - headlen = random.randint(1, 64) + head_size = self.get_head_size(buf, 30) + if len(buf) - head_size > 64: + headlen = head_size + random.randint(1, 64) else: headlen = len(buf) headdata = buf[:headlen] diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index 5450e7a2..a99fb0bd 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -22,6 +22,8 @@ import hashlib import logging +from shadowsocks.common import ord + def create_obfs(method): return plain(method) @@ -70,3 +72,15 @@ def server_post_decrypt(self, buf): def dispose(self): pass + def get_head_size(self, buf, def_value): + if len(buf) < 2: + return def_value + head_type = ord(buf[0]) & 0xF + if head_type == 1: + return 7 + if head_type == 4: + return 19 + if head_type == 3: + return 4 + ord(buf[1]) + return def_value + diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 03c73b22..16c83428 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -82,17 +82,6 @@ def server_encode(self, buf): def server_decode(self, buf): return (buf, True, False) - def get_head_size(self, buf, def_value): - if len(buf) < 2: - return def_value - if ord(buf[0]) == 1: - return 7 - if ord(buf[0]) == 4: - return 19 - if ord(buf[0]) == 3: - return 4 + ord(buf[1]) - return def_value - class verify_simple(verify_base): def __init__(self, method): super(verify_simple, self).__init__(method) From 125c3e88fdf34052eb1a9e330f270a6c7caba5df Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 12 Nov 2015 18:06:12 +0800 Subject: [PATCH 100/487] random size from 0 --- shadowsocks/obfsplugin/http_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 926ba857..9dce4400 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -92,7 +92,7 @@ def client_encode(self, buf): return buf head_size = self.get_head_size(buf, 30) if len(buf) - head_size > 64: - headlen = head_size + random.randint(1, 64) + headlen = head_size + random.randint(0, 64) else: headlen = len(buf) headdata = buf[:headlen] From 8826629741bd7ae7c574fff1bdd521392adb5646 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 16 Nov 2015 17:10:39 +0800 Subject: [PATCH 101/487] fix http_simple calc 'head_size' add 'iv', 'key', 'head_len' in server_info --- shadowsocks/common.py | 2 +- shadowsocks/encrypt.py | 2 ++ shadowsocks/local.py | 2 ++ shadowsocks/obfsplugin/http_simple.py | 2 +- shadowsocks/tcprelay.py | 25 +++++++++++++++++++++++-- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 7f306ea8..377f06a0 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -204,7 +204,7 @@ def parse_header(data): elif addrtype == ADDRTYPE_HOST: if len(data) > 2: addrlen = ord(data[1]) - if len(data) >= 2 + addrlen: + if len(data) >= 4 + addrlen: dest_addr = data[2:2 + addrlen] dest_port = struct.unpack('>H', data[2 + addrlen:4 + addrlen])[0] diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index d3b27527..24943957 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -78,6 +78,7 @@ def __init__(self, key, method): self.iv_sent = False self.cipher_iv = b'' self.iv_buf = b'' + self.cipher_key = b'' self.decipher = None method = method.lower() self._method_info = self.get_method_info(method) @@ -109,6 +110,7 @@ def get_cipher(self, password, method, op, iv): if op == 1: # this iv is for cipher not decipher self.cipher_iv = iv[:m[1]] + self.cipher_key = key return m[2](method, key, iv, op) def encrypt(self, buf): diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 096283c1..3b6523fc 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -47,6 +47,8 @@ def main(): asyncdns.IPV6_CONNECTION_SUPPORT = False daemon.daemon_exec(config) + logging.info("local start with protocol[%s] password [%s] method [%s] obfs [%s] obfs_param [%s]" % + (config['protocol'], config['password'], config['method'], config['obfs'], config['obfs_param'])) try: logging.info("starting local at %s:%d" % diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 9dce4400..7c0cf995 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -90,7 +90,7 @@ def encode_head(self, buf): def client_encode(self, buf): if self.has_sent_header: return buf - head_size = self.get_head_size(buf, 30) + head_size = len(self.server_info.iv) + self.server_info.head_len if len(buf) - head_size > 64: headlen = head_size + random.randint(0, 64) else: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 355b2111..82ffb09b 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -118,16 +118,22 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info = obfs.server_info(server.obfs_data) server_info.host = config['server'] server_info.port = server._listen_port - server_info.tcp_mss = 1440 server_info.param = config['obfs_param'] + server_info.iv = self._encryptor.cipher_iv + server_info.key = self._encryptor.cipher_key + server_info.head_len = 30 + server_info.tcp_mss = 1440 self._obfs.set_server_info(server_info) self._protocol = obfs.obfs(config['protocol']) server_info = obfs.server_info(server.protocol_data) server_info.host = config['server'] server_info.port = server._listen_port - server_info.tcp_mss = 1440 server_info.param = '' + server_info.iv = self._encryptor.cipher_iv + server_info.key = self._encryptor.cipher_key + server_info.head_len = 30 + server_info.tcp_mss = 1440 self._protocol.set_server_info(server_info) self._redir_list = config.get('redirect', ["0.0.0.0:0"]) @@ -386,6 +392,18 @@ def _handle_stage_connecting(self, data): traceback.print_exc() self.destroy() + def _get_head_size(self, buf, def_value): + if len(buf) < 2: + return def_value + head_type = ord(buf[0]) & 0xF + if head_type == 1: + return 7 + if head_type == 4: + return 19 + if head_type == 3: + return 4 + ord(buf[1]) + return def_value + def _handle_stage_addr(self, ogn_data, data): try: if self._is_local: @@ -439,6 +457,9 @@ def _handle_stage_addr(self, ogn_data, data): self._write_to_sock((b'\x05\x00\x00\x01' b'\x00\x00\x00\x00\x10\x10'), self._local_sock) + head_len = self._get_head_size(data, 30) + self._obfs.obfs.server_info.head_len = head_len + self._protocol.obfs.server_info.head_len = head_len if CLIENT_NEW_PROTOCOL: rnd_len = random.randint(1, 32) total_len = 7 + rnd_len + len(data) From 737b7d9890345f398c26c25fca328e64264a0637 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 Nov 2015 15:42:59 +0800 Subject: [PATCH 102/487] add verify_sha1 (libev OTA) --- shadowsocks/obfs.py | 5 +- shadowsocks/obfsplugin/auth.py | 326 ++++++++++++++++++++++++ shadowsocks/obfsplugin/verify_simple.py | 256 ++++--------------- shadowsocks/tcprelay.py | 17 +- 4 files changed, 386 insertions(+), 218 deletions(-) create mode 100644 shadowsocks/obfsplugin/auth.py diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 1752a56e..1bfaf673 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -23,13 +23,14 @@ import logging from shadowsocks import common -from shadowsocks.obfsplugin import plain, http_simple, verify_simple +from shadowsocks.obfsplugin import plain, http_simple, verify, auth method_supported = {} method_supported.update(plain.obfs_map) method_supported.update(http_simple.obfs_map) -method_supported.update(verify_simple.obfs_map) +method_supported.update(verify.obfs_map) +method_supported.update(auth.obfs_map) class server_info(object): def __init__(self, data): diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py new file mode 100644 index 00000000..0ed32526 --- /dev/null +++ b/shadowsocks/obfsplugin/auth.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python +# +# Copyright 2015-2015 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import hashlib +import logging +import binascii +import base64 +import time +import datetime +import random +import struct +import zlib +import hmac +import hashlib + +import shadowsocks +from shadowsocks import common +from shadowsocks.obfsplugin import plain +from shadowsocks.common import to_bytes, to_str, ord, chr + +def create_auth_obfs(method): + return auth_simple(method) + +obfs_map = { + 'auth_simple': (create_auth_obfs,), +} + +def match_begin(str1, str2): + if len(str1) >= len(str2): + if str1[:len(str2)] == str2: + return True + return False + +class obfs_verify_data(object): + def __init__(self): + pass + +class verify_base(plain.plain): + def __init__(self, method): + super(verify_base, self).__init__(method) + self.method = method + + def init_data(self): + return '' + + def set_server_info(self, server_info): + self.server_info = server_info + + def client_encode(self, buf): + return buf + + def client_decode(self, buf): + return (buf, False) + + def server_encode(self, buf): + return buf + + def server_decode(self, buf): + return (buf, True, False) + +class client_queue(object): + def __init__(self, begin_id): + self.front = begin_id + self.back = begin_id + self.alloc = {} + self.enable = True + self.last_update = time.time() + + def update(self): + self.last_update = time.time() + + def is_active(self): + return time.time() - self.last_update < 60 * 3 + + def re_enable(self, connection_id): + self.enable = True + self.alloc = {} + self.front = connection_id + self.back = connection_id + + def insert(self, connection_id): + self.update() + if not self.enable: + logging.warn('auth_simple: not enable') + return False + if connection_id < self.front: + logging.warn('auth_simple: duplicate id') + return False + if not self.is_active(): + self.re_enable(connection_id) + if connection_id > self.front + 0x4000: + logging.warn('auth_simple: wrong id') + return False + if connection_id in self.alloc: + logging.warn('auth_simple: duplicate id 2') + return False + if self.back <= connection_id: + self.back = connection_id + 1 + self.alloc[connection_id] = 1 + while (self.front in self.alloc) or self.front + 0x1000 < self.back: + if self.front in self.alloc: + del self.alloc[self.front] + self.front += 1 + return True + +class obfs_auth_data(object): + def __init__(self): + self.client_id = {} + self.startup_time = int(time.time() - 30) & 0xFFFFFFFF + self.local_client_id = b'' + self.connection_id = 0 + self.max_client = 16 # max active client count + self.max_buffer = max(self.max_client, 256) # max client id buffer size + + def update(self, client_id, connection_id): + if client_id in self.client_id: + self.client_id[client_id].update() + + def insert(self, client_id, connection_id): + if client_id not in self.client_id or not self.client_id[client_id].enable: + active = 0 + for c_id in self.client_id: + if self.client_id[c_id].is_active(): + active += 1 + if active >= self.max_client: + logging.warn('auth_simple: max active clients exceeded') + return False + + if len(self.client_id) < self.max_client: + if client_id not in self.client_id: + self.client_id[client_id] = client_queue(connection_id) + else: + self.client_id[client_id].re_enable(connection_id) + return self.client_id[client_id].insert(connection_id) + keys = self.client_id.keys() + random.shuffle(keys) + for c_id in keys: + if not self.client_id[c_id].is_active() and self.client_id[c_id].enable: + if len(self.client_id) >= self.max_buffer: + del self.client_id[c_id] + else: + self.client_id[c_id].enable = False + if client_id not in self.client_id: + self.client_id[client_id] = client_queue(connection_id) + else: + self.client_id[client_id].re_enable(connection_id) + return self.client_id[client_id].insert(connection_id) + logging.warn('auth_simple: no inactive client [assert]') + return False + else: + return self.client_id[client_id].insert(connection_id) + +class auth_simple(verify_base): + def __init__(self, method): + super(auth_simple, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + self.max_time_dif = 60 * 5 # time dif (second) setting + + def init_data(self): + return obfs_auth_data() + + def pack_data(self, buf): + if len(buf) == 0: + return b'' + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) + data = common.chr(len(rnd_data) + 1) + rnd_data + buf + data = struct.pack('>H', len(data) + 6) + data + crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff + data += struct.pack(' 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data uncorrect CRC32') + + pos = common.ord(self.recv_buf[2]) + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf + + def server_pre_encrypt(self, buf): + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + logging.info('auth_simple: over size') + return b'E' + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: + logging.info('auth_simple: crc32 error, data %s' % (binascii.hexlify(self.recv_buf[:length]),)) + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return b'E' + else: + raise Exception('server_post_decrype data uncorrect CRC32') + + pos = common.ord(self.recv_buf[2]) + 2 + out_buf += self.recv_buf[pos:length - 4] + if not self.has_recv_header: + if len(out_buf) < 12: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_simple: too short') + return b'E' + utc_time = struct.unpack(' self.max_time_dif \ + or common.int32(utc_time - self.server_info.data.startup_time) < 0: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_simple: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) + return b'E' + elif self.server_info.data.insert(client_id, connection_id): + self.has_recv_header = True + out_buf = out_buf[12:] + self.client_id = client_id + self.connection_id = connection_id + else: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_simple: auth fail, data %s' % (binascii.hexlify(out_buf),)) + return b'E' + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.server_info.data.update(self.client_id, self.connection_id) + self.decrypt_packet_num += 1 + return out_buf + diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify_simple.py index 16c83428..8b513b87 100644 --- a/shadowsocks/obfsplugin/verify_simple.py +++ b/shadowsocks/obfsplugin/verify_simple.py @@ -28,11 +28,13 @@ import random import struct import zlib +import hmac +import hashlib import shadowsocks from shadowsocks import common from shadowsocks.obfsplugin import plain -from shadowsocks.common import to_bytes, to_str, ord +from shadowsocks.common import to_bytes, to_str, ord, chr def create_verify_obfs(method): return verify_simple(method) @@ -40,13 +42,17 @@ def create_verify_obfs(method): def create_verify_deflate(method): return verify_deflate(method) +def create_verify_sha1(method): + return verify_sha1(method) + def create_auth_obfs(method): return auth_simple(method) obfs_map = { 'verify_simple': (create_verify_obfs,), 'verify_deflate': (create_verify_deflate,), - 'auth_simple': (create_auth_obfs,), + 'verify_sha1': (create_verify_sha1,), + 'verify_sha1_compatible': (create_verify_sha1,), } def match_begin(str1, str2): @@ -261,143 +267,35 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return out_buf -class client_queue(object): - def __init__(self, begin_id): - self.front = begin_id - self.back = begin_id - self.alloc = {} - self.enable = True - self.last_update = time.time() - - def update(self): - self.last_update = time.time() - - def is_active(self): - return time.time() - self.last_update < 60 * 3 - - def re_enable(self, connection_id): - self.enable = True - self.alloc = {} - self.front = connection_id - self.back = connection_id - - def insert(self, connection_id): - self.update() - if not self.enable: - logging.warn('auth_simple: not enable') - return False - if connection_id < self.front: - logging.warn('auth_simple: duplicate id') - return False - if not self.is_active(): - self.re_enable(connection_id) - if connection_id > self.front + 0x4000: - logging.warn('auth_simple: wrong id') - return False - if connection_id in self.alloc: - logging.warn('auth_simple: duplicate id 2') - return False - if self.back <= connection_id: - self.back = connection_id + 1 - self.alloc[connection_id] = 1 - while (self.front in self.alloc) or self.front + 0x1000 < self.back: - if self.front in self.alloc: - del self.alloc[self.front] - self.front += 1 - return True - -class obfs_auth_data(object): - def __init__(self): - self.client_id = {} - self.startup_time = int(time.time() - 30) & 0xFFFFFFFF - self.local_client_id = b'' - self.connection_id = 0 - self.max_client = 16 # max active client count - self.max_buffer = max(self.max_client, 256) # max client id buffer size - - def update(self, client_id, connection_id): - if client_id in self.client_id: - self.client_id[client_id].update() - - def insert(self, client_id, connection_id): - if client_id not in self.client_id or not self.client_id[client_id].enable: - active = 0 - for c_id in self.client_id: - if self.client_id[c_id].is_active(): - active += 1 - if active >= self.max_client: - logging.warn('auth_simple: max active clients exceeded') - return False - - if len(self.client_id) < self.max_client: - if client_id not in self.client_id: - self.client_id[client_id] = client_queue(connection_id) - else: - self.client_id[client_id].re_enable(connection_id) - return self.client_id[client_id].insert(connection_id) - keys = self.client_id.keys() - random.shuffle(keys) - for c_id in keys: - if not self.client_id[c_id].is_active() and self.client_id[c_id].enable: - if len(self.client_id) >= self.max_buffer: - del self.client_id[c_id] - else: - self.client_id[c_id].enable = False - if client_id not in self.client_id: - self.client_id[client_id] = client_queue(connection_id) - else: - self.client_id[client_id].re_enable(connection_id) - return self.client_id[client_id].insert(connection_id) - logging.warn('auth_simple: no inactive client [assert]') - return False - else: - return self.client_id[client_id].insert(connection_id) - -class auth_simple(verify_base): +class verify_sha1(verify_base): def __init__(self, method): - super(auth_simple, self).__init__(method) + super(verify_sha1, self).__init__(method) self.recv_buf = b'' self.unit_len = 8100 - self.decrypt_packet_num = 0 self.raw_trans = False + self.pack_id = 0 + self.recv_id = 0 self.has_sent_header = False self.has_recv_header = False - self.client_id = 0 - self.connection_id = 0 - self.max_time_dif = 60 * 5 # time dif (second) setting - - def init_data(self): - return obfs_auth_data() def pack_data(self, buf): if len(buf) == 0: return b'' - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) - data = common.chr(len(rnd_data) + 1) + rnd_data + buf - data = struct.pack('>H', len(data) + 6) + data - crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff - data += struct.pack('I', self.pack_id), buf, hashlib.sha1).digest() + data = struct.pack('>H', len(buf)) + sha1data[:10] + buf + self.pack_id += 1 return data - def auth_data(self): - utc_time = int(time.time()) & 0xFFFFFFFF - if self.server_info.data.connection_id > 0xFF000000: - self.server_info.data.local_client_id = b'' - if not self.server_info.data.local_client_id: - self.server_info.data.local_client_id = os.urandom(4) - logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) - self.server_info.data.connection_id = struct.unpack(' self.unit_len: @@ -407,105 +305,49 @@ def client_pre_encrypt(self, buf): return ret def client_post_decrypt(self, buf): - if self.raw_trans: - return buf - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('server_post_decrype data error') - if length > len(self.recv_buf): - break - - if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('server_post_decrype data uncorrect CRC32') - - pos = common.ord(self.recv_buf[2]) + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - - if out_buf: - self.decrypt_packet_num += 1 - return out_buf + return buf def server_pre_encrypt(self, buf): - ret = b'' - while len(buf) > self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret + return buf def server_post_decrypt(self, buf): if self.raw_trans: return buf self.recv_buf += buf out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - logging.info('auth_simple: over size') + if not self.has_recv_header: + if len(self.recv_buf) < 2: + return b'' + if (ord(self.recv_buf[0]) & 0x10) != 0x10: + if self.method == 'verify_sha1': + logging.error('Not One-time authentication header') return b'E' else: - raise Exception('server_post_decrype data error') + self.raw_trans = True + return self.recv_buf + head_size = self.get_head_size(self.recv_buf, 30) + if len(self.recv_buf) < head_size + 10: + return b'' + sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:head_size], hashlib.sha1).digest()[:10] + if sha1data != self.recv_buf[head_size:head_size + 10]: + logging.error('server_post_decrype data uncorrect auth HMAC-SHA1') + return b'E' + out_buf = to_bytes(chr(ord(self.recv_buf[0]) & 0xF)) + self.recv_buf[1:head_size] + self.recv_buf = self.recv_buf[head_size + 10:] + self.has_recv_header = True + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + 12 if length > len(self.recv_buf): break - if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: - logging.info('auth_simple: crc32 error, data %s' % (binascii.hexlify(self.recv_buf[:length]),)) - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return b'E' - else: - raise Exception('server_post_decrype data uncorrect CRC32') + data = self.recv_buf[12:length] + sha1data = hmac.new(self.server_info.recv_iv + struct.pack('>I', self.recv_id), data, hashlib.sha1).digest()[:10] + if sha1data != self.recv_buf[2:12]: + raise Exception('server_post_decrype data uncorrect chunk HMAC-SHA1') - pos = common.ord(self.recv_buf[2]) + 2 - out_buf += self.recv_buf[pos:length - 4] - if not self.has_recv_header: - if len(out_buf) < 12: - self.raw_trans = True - self.recv_buf = b'' - logging.info('auth_simple: too short') - return b'E' - utc_time = struct.unpack(' self.max_time_dif \ - or common.int32(utc_time - self.server_info.data.startup_time) < 0: - self.raw_trans = True - self.recv_buf = b'' - logging.info('auth_simple: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) - return b'E' - elif self.server_info.data.insert(client_id, connection_id): - self.has_recv_header = True - out_buf = out_buf[12:] - self.client_id = client_id - self.connection_id = connection_id - else: - self.raw_trans = True - self.recv_buf = b'' - logging.info('auth_simple: auth fail, data %s' % (binascii.hexlify(out_buf),)) - return b'E' + self.recv_id += 1 + out_buf += data self.recv_buf = self.recv_buf[length:] - if out_buf: - self.server_info.data.update(self.client_id, self.connection_id) - self.decrypt_packet_num += 1 return out_buf diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 82ffb09b..93ff9ffb 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -30,9 +30,6 @@ from shadowsocks import encrypt, obfs, eventloop, shell, common from shadowsocks.common import pre_parse_header, parse_header -# set it 'True' if run as a local client and connect to a server which support new protocol -CLIENT_NEW_PROTOCOL = False #deprecated - # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 @@ -120,6 +117,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.port = server._listen_port server_info.param = config['obfs_param'] server_info.iv = self._encryptor.cipher_iv + server_info.recv_iv = b'' server_info.key = self._encryptor.cipher_key server_info.head_len = 30 server_info.tcp_mss = 1440 @@ -131,6 +129,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.port = server._listen_port server_info.param = '' server_info.iv = self._encryptor.cipher_iv + server_info.recv_iv = b'' server_info.key = self._encryptor.cipher_key server_info.head_len = 30 server_info.tcp_mss = 1440 @@ -460,12 +459,6 @@ def _handle_stage_addr(self, ogn_data, data): head_len = self._get_head_size(data, 30) self._obfs.obfs.server_info.head_len = head_len self._protocol.obfs.server_info.head_len = head_len - if CLIENT_NEW_PROTOCOL: - rnd_len = random.randint(1, 32) - total_len = 7 + rnd_len + len(data) - data = b'\x88' + struct.pack('>H', total_len) + chr(rnd_len) + (b' ' * (rnd_len - 1)) + data - crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff - data += struct.pack(' Date: Tue, 17 Nov 2015 15:43:45 +0800 Subject: [PATCH 103/487] rename verify.py --- shadowsocks/obfsplugin/{verify_simple.py => verify.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename shadowsocks/obfsplugin/{verify_simple.py => verify.py} (100%) diff --git a/shadowsocks/obfsplugin/verify_simple.py b/shadowsocks/obfsplugin/verify.py similarity index 100% rename from shadowsocks/obfsplugin/verify_simple.py rename to shadowsocks/obfsplugin/verify.py From 19c84cac175977edca97570899b3ee2c9825b7d9 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 Nov 2015 16:55:15 +0800 Subject: [PATCH 104/487] avoid conflict with OTA --- shadowsocks/common.py | 4 ++-- shadowsocks/obfsplugin/verify.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 377f06a0..ddbbb602 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -192,8 +192,8 @@ def parse_header(data): dest_addr = None dest_port = None header_length = 0 - connecttype = (addrtype & 0x10) and 1 or 0 - addrtype &= ~0x10 + connecttype = (addrtype & 0x8) and 1 or 0 + addrtype &= ~0x8 if addrtype == ADDRTYPE_IPV4: if len(data) >= 7: dest_addr = socket.inet_ntoa(data[1:5]) diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 8b513b87..1dfbdb02 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -286,8 +286,8 @@ def pack_data(self, buf): self.pack_id += 1 return data - def auth_pack_data(self, buf): - data = buf + def pack_auth_data(self, buf): + data = chr(ord(buf[0]) | 0x10) + buf[1:] data += hmac.new(self.server_info.iv + self.server_info.key, buf, hashlib.sha1).digest()[:10] return data @@ -295,7 +295,7 @@ def client_pre_encrypt(self, buf): ret = b'' if not self.has_sent_header: datalen = self.get_head_size(buf, 30) - ret += self.auth_pack_data(buf[datalen:]) + ret += self.pack_auth_data(buf[datalen:]) buf = buf[datalen:] self.has_sent_header = True while len(buf) > self.unit_len: From 84275b681565b13c002720c56d9be4abb6e065da Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 Nov 2015 17:39:18 +0800 Subject: [PATCH 105/487] UDP over TCP use 0x8 as flag --- shadowsocks/obfsplugin/plain.py | 2 +- shadowsocks/obfsplugin/verify.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index a99fb0bd..5bba57ca 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -75,7 +75,7 @@ def dispose(self): def get_head_size(self, buf, def_value): if len(buf) < 2: return def_value - head_type = ord(buf[0]) & 0xF + head_type = ord(buf[0]) & 0x7 if head_type == 1: return 7 if head_type == 4: diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 1dfbdb02..770ce295 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -288,7 +288,7 @@ def pack_data(self, buf): def pack_auth_data(self, buf): data = chr(ord(buf[0]) | 0x10) + buf[1:] - data += hmac.new(self.server_info.iv + self.server_info.key, buf, hashlib.sha1).digest()[:10] + data += hmac.new(self.server_info.iv + self.server_info.key, data, hashlib.sha1).digest()[:10] return data def client_pre_encrypt(self, buf): From 3930cfc72653cde212f9b95efd2fcd62bb4749c2 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 Nov 2015 17:49:04 +0800 Subject: [PATCH 106/487] fix a bug --- shadowsocks/obfsplugin/verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 770ce295..1c939ae1 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -295,7 +295,7 @@ def client_pre_encrypt(self, buf): ret = b'' if not self.has_sent_header: datalen = self.get_head_size(buf, 30) - ret += self.pack_auth_data(buf[datalen:]) + ret += self.pack_auth_data(buf[:datalen]) buf = buf[datalen:] self.has_sent_header = True while len(buf) > self.unit_len: From 0504fb43cf24dd804d4d6e810771e422d087e458 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 19 Nov 2015 22:21:59 +0800 Subject: [PATCH 107/487] fix a ipv6 bug in mysql mode --- server_pool.py | 3 +++ shadowsocks/obfsplugin/verify.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server_pool.py b/server_pool.py index 4b4663c0..002ddfe1 100644 --- a/server_pool.py +++ b/server_pool.py @@ -52,6 +52,9 @@ def __init__(self): self.config = shell.get_config(False) shell.print_shadowsocks() self.dns_resolver = asyncdns.DNSResolver() + if not self.config.get('dns_ipv6', False): + asyncdns.IPV6_CONNECTION_SUPPORT = False + self.mgr = asyncmgr.ServerMgr() self.tcp_servers_pool = {} diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 1c939ae1..5c54c781 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -332,7 +332,7 @@ def server_post_decrypt(self, buf): if sha1data != self.recv_buf[head_size:head_size + 10]: logging.error('server_post_decrype data uncorrect auth HMAC-SHA1') return b'E' - out_buf = to_bytes(chr(ord(self.recv_buf[0]) & 0xF)) + self.recv_buf[1:head_size] + out_buf = to_bytes(chr(ord(self.recv_buf[0]) & 0xEF)) + self.recv_buf[1:head_size] self.recv_buf = self.recv_buf[head_size + 10:] self.has_recv_header = True while len(self.recv_buf) > 2: @@ -345,7 +345,7 @@ def server_post_decrypt(self, buf): if sha1data != self.recv_buf[2:12]: raise Exception('server_post_decrype data uncorrect chunk HMAC-SHA1') - self.recv_id += 1 + self.recv_id = (self.recv_id + 1) & 0xFFFFFFFF out_buf += data self.recv_buf = self.recv_buf[length:] From 57433bccb9d5d9c6b51bd281d79b04975acfa251 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 25 Nov 2015 16:38:08 +0800 Subject: [PATCH 108/487] fix transfer upload size calculation. avoid flow attack --- shadowsocks/tcprelay.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 93ff9ffb..e9492f05 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -275,7 +275,9 @@ def _write_to_sock(self, data, sock): if self._is_local: pass else: - if sock == self._local_sock and self._encrypt_correct and (self._obfs is not None): + if sock == self._remote_sock: + self._server.server_transfer_ul += len(data) + elif self._encrypt_correct and (self._obfs is not None): obfs_encode = self._obfs.server_encode(data) data = obfs_encode if data: @@ -614,7 +616,6 @@ def _on_local_read(self): return if not data: return - self._server.server_transfer_ul += len(data) if self._stage == STAGE_STREAM: if self._is_local: if self._encryptor is not None: From 4b9741256912f66997ac7c79c994e909529bf4e9 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 25 Nov 2015 16:58:30 +0800 Subject: [PATCH 109/487] only correct request add into transfer size --- shadowsocks/tcprelay.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e9492f05..c24f2b6c 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -276,7 +276,8 @@ def _write_to_sock(self, data, sock): pass else: if sock == self._remote_sock: - self._server.server_transfer_ul += len(data) + if self._encrypt_correct: + self._server.server_transfer_ul += len(data) elif self._encrypt_correct and (self._obfs is not None): obfs_encode = self._obfs.server_encode(data) data = obfs_encode From 409f3b053dafb0428c14d9eb3f179e266e1c6aa0 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 27 Nov 2015 15:42:22 +0800 Subject: [PATCH 110/487] fix ipv6 check --- shadowsocks/asyncdns.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index d958752e..4a34b6bb 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -80,8 +80,8 @@ def detect_ipv6_supprot(): if 'has_ipv6' in dir(socket): - s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) try: + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) s.connect(('ipv6.google.com', 0)) print('IPv6 support') return True @@ -447,6 +447,15 @@ def resolve(self, hostname, callback): if not is_valid_hostname(hostname): callback(None, Exception('invalid hostname: %s' % hostname)) return + if False: + addrs = socket.getaddrinfo(hostname, 0, 0, + socket.SOCK_DGRAM, socket.SOL_UDP) + if addrs: + af, socktype, proto, canonname, sa = addrs[0] + logging.debug('DNS resolve %s %s' % (hostname, sa[0]) ) + self._cache[hostname] = sa[0] + callback((hostname, sa[0]), None) + return arr = self._hostname_to_cb.get(hostname, None) if not arr: if IPV6_CONNECTION_SUPPORT: From 4894925cacb52edb8a5e6a5b0f863a97a4a6207a Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 27 Nov 2015 16:47:08 +0800 Subject: [PATCH 111/487] add demo code in switchrule.py --- switchrule.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/switchrule.py b/switchrule.py index 036c61bb..3721d2c9 100644 --- a/switchrule.py +++ b/switchrule.py @@ -1,6 +1,8 @@ def getKeys(): return ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] + #return ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable', 'plan' ] # append the column name 'plan' def isTurnOn(row): return True + #return row['plan'] == 'B' # then judge here From b5da4bc86a4fee5dbb61b7ca748df47d510cbf85 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 3 Dec 2015 16:48:08 +0800 Subject: [PATCH 112/487] add MYSQL_TRANSFER_MUL add protocol_param (for auth_simple) --- Config.py | 1 + db_transfer.py | 10 ++++++---- shadowsocks/obfsplugin/auth.py | 14 ++++++++++++-- shadowsocks/obfsplugin/http_simple.py | 8 ++++---- shadowsocks/server.py | 5 ++++- shadowsocks/shell.py | 11 +++++++++-- shadowsocks/tcprelay.py | 9 ++++++--- 7 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Config.py b/Config.py index 7a444e6f..6323abe4 100644 --- a/Config.py +++ b/Config.py @@ -4,6 +4,7 @@ MYSQL_USER = 'ss' MYSQL_PASS = 'ss' MYSQL_DB = 'shadowsocks' +MYSQL_TRANSFER_MUL = 1.0 MANAGE_PASS = 'ss233333333' #if you want manage in other server you should set this value to global ip diff --git a/db_transfer.py b/db_transfer.py index ba6e3eab..caf60f3b 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -37,14 +37,16 @@ def push_db_all_user(self): continue elif last_transfer[id][0] <= curr_transfer[id][0] and \ last_transfer[id][1] <= curr_transfer[id][1]: - dt_transfer[id] = [curr_transfer[id][0] - last_transfer[id][0], - curr_transfer[id][1] - last_transfer[id][1]] + dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * Config.MYSQL_TRANSFER_MUL), + int((curr_transfer[id][1] - last_transfer[id][1]) * Config.MYSQL_TRANSFER_MUL)] else: - dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + dt_transfer[id] = [int(curr_transfer[id][0] * Config.MYSQL_TRANSFER_MUL), + int(curr_transfer[id][1] * Config.MYSQL_TRANSFER_MUL)] else: if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: continue - dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + dt_transfer[id] = [int(curr_transfer[id][0] * Config.MYSQL_TRANSFER_MUL), + int(curr_transfer[id][1] * Config.MYSQL_TRANSFER_MUL)] self.last_get_transfer = curr_transfer query_head = 'UPDATE user' diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 0ed32526..5ffca95b 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -127,13 +127,16 @@ def __init__(self): self.startup_time = int(time.time() - 30) & 0xFFFFFFFF self.local_client_id = b'' self.connection_id = 0 - self.max_client = 16 # max active client count - self.max_buffer = max(self.max_client, 256) # max client id buffer size + self.set_max_client(16) # max active client count def update(self, client_id, connection_id): if client_id in self.client_id: self.client_id[client_id].update() + def set_max_client(self, max_client): + self.max_client = max_client + self.max_buffer = max(self.max_client * 2, 256) + def insert(self, client_id, connection_id): if client_id not in self.client_id or not self.client_id[client_id].enable: active = 0 @@ -184,6 +187,13 @@ def __init__(self, method): def init_data(self): return obfs_auth_data() + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param) + except: + pass + def pack_data(self, buf): if len(buf) == 0: return b'' diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 7c0cf995..2a7a1071 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -101,7 +101,7 @@ def client_encode(self, buf): if self.server_info.port != 80: port = b':' + common.to_bytes(str(self.server_info.port)) http_head = b"GET /" + self.encode_head(headdata) + b" HTTP/1.1\r\n" - http_head += b"Host: " + (self.server_info.param or self.server_info.host) + port + b"\r\n" + http_head += b"Host: " + (self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\nDNT: 1\r\nConnection: keep-alive\r\n\r\n" self.has_sent_header = True @@ -171,8 +171,8 @@ def server_decode(self, buf): else: return (b'', True, False) - datas = buf.split(b'\r\n\r\n', 1) - if datas: + if b'\r\n\r\n' in buf: + datas = buf.split(b'\r\n\r\n', 1) ret_buf = self.get_data_from_http_header(buf) if len(datas) > 1: ret_buf += datas[1] @@ -205,7 +205,7 @@ def client_encode(self, buf): port = b':' + common.to_bytes(str(self.server_info.port)) self.has_sent_header = True http_head = b"GET / HTTP/1.1\r\n" - http_head += b"Host: " + (self.server_info.param or self.server_info.host) + port + b"\r\n" + http_head += b"Host: " + (self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" http_head += b"Connection: Upgrade, HTTP2-Settings\r\nUpgrade: h2c\r\n" http_head += b"HTTP2-Settings: " + base64.urlsafe_b64encode(buf) + b"\r\n" return http_head + b"\r\n" diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 4c19474f..a3a93273 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -69,6 +69,8 @@ def main(): del config['port_password'] for port, password_obfs in port_password.items(): protocol = config.get("protocol", 'origin') + protocol_param = config.get("protocol_param", '') + obfs = config.get("obfs", 'plain') obfs_param = config.get("obfs_param", '') if type(password_obfs) == list: password = password_obfs[0] @@ -80,7 +82,6 @@ def main(): obfs_param = password_obfs.get('obfs_param', '') else: password = password_obfs - obfs = config["obfs"] a_config = config.copy() ipv6_ok = False logging.info("server start with protocol[%s] password [%s] method [%s] obfs [%s] obfs_param [%s]" % @@ -92,6 +93,7 @@ def main(): a_config['server_port'] = int(port) a_config['password'] = password a_config['protocol'] = protocol + a_config['protocol_param'] = protocol_param a_config['obfs'] = obfs a_config['obfs_param'] = obfs_param a_config['server'] = a_config['server_ipv6'] @@ -109,6 +111,7 @@ def main(): a_config['server_port'] = int(port) a_config['password'] = password a_config['protocol'] = protocol + a_config['protocol_param'] = protocol_param a_config['obfs'] = obfs a_config['obfs_param'] = obfs_param logging.info("starting server at %s:%d" % diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 38d2432b..2d636bfd 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -130,11 +130,11 @@ def get_config(is_local): logging.basicConfig(level=logging.INFO, format='%(levelname)-s: %(message)s') if is_local: - shortopts = 'hd:s:b:p:k:l:m:o:c:t:vq' + shortopts = 'hd:s:b:p:k:l:m:P:o:G:g:c:t:vq' longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'user=', 'version'] else: - shortopts = 'hd:s:p:k:m:o:c:t:vq' + shortopts = 'hd:s:p:k:m:P:o:G:g:c:t:vq' longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'workers=', 'forbidden-ip=', 'user=', 'manager-address=', 'version'] try: @@ -168,8 +168,14 @@ def get_config(is_local): config['server'] = to_str(value) elif key == '-m': config['method'] = to_str(value) + elif key == '-P': + config['protocol'] = to_str(value) elif key == '-o': config['obfs'] = to_str(value) + elif key == '-G': + config['protocol_param'] = to_str(value) + elif key == '-g': + config['obfs_param'] = to_str(value) elif key == '-b': config['local_address'] = to_str(value) elif key == '-v': @@ -219,6 +225,7 @@ def get_config(is_local): config['password'] = to_bytes(config.get('password', b'')) config['method'] = to_str(config.get('method', 'aes-256-cfb')) config['protocol'] = to_str(config.get('protocol', 'origin')) + config['protocol_param'] = to_str(config.get('protocol_param', '')) config['obfs'] = to_str(config.get('obfs', 'plain')) config['obfs_param'] = to_str(config.get('obfs_param', '')) config['port_password'] = config.get('port_password', None) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c24f2b6c..8953f218 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -115,7 +115,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info = obfs.server_info(server.obfs_data) server_info.host = config['server'] server_info.port = server._listen_port - server_info.param = config['obfs_param'] + server_info.protocol_param = '' + server_info.obfs_param = config['obfs_param'] server_info.iv = self._encryptor.cipher_iv server_info.recv_iv = b'' server_info.key = self._encryptor.cipher_key @@ -127,7 +128,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info = obfs.server_info(server.protocol_data) server_info.host = config['server'] server_info.port = server._listen_port - server_info.param = '' + server_info.protocol_param = config['protocol_param'] + server_info.obfs_param = '' server_info.iv = self._encryptor.cipher_iv server_info.recv_iv = b'' server_info.key = self._encryptor.cipher_key @@ -343,11 +345,12 @@ def _get_redirect_host(self, client_address, ogn_data): return (host_post, 80) def _handel_protocol_error(self, client_address, ogn_data): - #raise Exception('can not parse header') logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1])) self._encrypt_correct = False #create redirect or disconnect by hash code host, port = self._get_redirect_host(client_address, ogn_data) + if port == 0: + raise Exception('can not parse header') data = b"\x03" + common.chr(len(host)) + common.to_bytes(host) + struct.pack('>H', port) logging.warn("TCP data redir %s:%d %s" % (host, port, binascii.hexlify(data))) return data + ogn_data From 96495e7dd99375190316841f23863d376bea917d Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 3 Dec 2015 21:33:39 +0800 Subject: [PATCH 113/487] add auth_sha1 --- config.json | 2 +- shadowsocks/obfsplugin/auth.py | 210 ++++++++++++++++++++++++++++++++- 2 files changed, 208 insertions(+), 4 deletions(-) diff --git a/config.json b/config.json index 3be6e567..0f2a90ff 100644 --- a/config.json +++ b/config.json @@ -7,7 +7,7 @@ "password": "m", "timeout": 120, "method": "aes-256-cfb", - "protocol": "origin", + "protocol": "auth_sha1_compatible", "obfs": "http_simple_compatible", "obfs_param": "", "dns_ipv6": false, diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 5ffca95b..2260bb41 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -36,11 +36,16 @@ from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord, chr -def create_auth_obfs(method): +def create_auth_simple(method): return auth_simple(method) +def create_auth_sha1(method): + return auth_sha1(method) + obfs_map = { - 'auth_simple': (create_auth_obfs,), + 'auth_simple': (create_auth_simple,), + 'auth_sha1': (create_auth_sha1,), + 'auth_sha1_compatible': (create_auth_sha1,), } def match_begin(str1, str2): @@ -192,7 +197,8 @@ def set_server_info(self, server_info): try: max_client = int(server_info.protocol_param) except: - pass + max_client = 16 + self.server_info.data.set_max_client(max_client) def pack_data(self, buf): if len(buf) == 0: @@ -334,3 +340,201 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return out_buf +class auth_sha1(verify_base): + def __init__(self, method): + super(auth_sha1, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + self.max_time_dif = 60 * 60 # time dif (second) setting + + def init_data(self): + return obfs_auth_data() + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + + def pack_data(self, buf): + if len(buf) == 0: + return b'' + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) + data = common.chr(len(rnd_data) + 1) + rnd_data + buf + data = struct.pack('>H', len(data) + 6) + data + adler32 = zlib.adler32(data) & 0xFFFFFFFF + data += struct.pack('H', len(data) + 10) + data + crc = binascii.crc32(self.server_info.key) + data = struct.pack(' 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + return None + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if zlib.adler32(self.recv_buf[:length - 4]) != struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + if not self.has_recv_header: + if len(self.recv_buf) < 4: + return b'' + crc = struct.pack('H', self.recv_buf[4:6])[0] + if length > len(self.recv_buf): + return b'' + sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] + if sha1data != self.recv_buf[length - 10:length]: + logging.error('server_post_decrype data uncorrect auth HMAC-SHA1') + return b'E' + pos = common.ord(self.recv_buf[6]) + 6 + out_buf = self.recv_buf[pos:length - 10] + if len(out_buf) < 12: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_sha1: too short') + return b'E' + utc_time = struct.unpack(' self.max_time_dif \ + or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_sha1: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) + return b'E' + elif self.server_info.data.insert(client_id, connection_id): + self.has_recv_header = True + out_buf = out_buf[12:] + self.client_id = client_id + self.connection_id = connection_id + else: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_sha1: auth fail, data %s' % (binascii.hexlify(out_buf),)) + return b'E' + self.recv_buf = self.recv_buf[length:] + self.has_recv_header = True + + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + logging.info('auth_sha1: over size') + return b'E' + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if zlib.adler32(self.recv_buf[:length - 4]) & 0xFFFFFFFF != struct.unpack(' Date: Mon, 7 Dec 2015 13:33:42 +0800 Subject: [PATCH 114/487] add protocol_param --- config.json | 1 + shadowsocks/tcprelay.py | 19 ++++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/config.json b/config.json index 0f2a90ff..d4ffc7c3 100644 --- a/config.json +++ b/config.json @@ -8,6 +8,7 @@ "timeout": 120, "method": "aes-256-cfb", "protocol": "auth_sha1_compatible", + "protocol_param": "", "obfs": "http_simple_compatible", "obfs_param": "", "dns_ipv6": false, diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8953f218..1f301aa6 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -274,15 +274,13 @@ def _write_to_sock(self, data, sock): return True else: try: - if self._is_local: - pass - else: + if self._encrypt_correct: if sock == self._remote_sock: - if self._encrypt_correct: - self._server.server_transfer_ul += len(data) - elif self._encrypt_correct and (self._obfs is not None): - obfs_encode = self._obfs.server_encode(data) - data = obfs_encode + self._server.server_transfer_ul += len(data) + self._update_activity(len(data)) + elif not self._is_local and self._obfs is not None: + obfs_encode = self._obfs.server_encode(data) + data = obfs_encode if data: l = len(data) s = sock.send(data) @@ -597,7 +595,6 @@ def _on_local_read(self): self.destroy() return ogn_data = data - self._update_activity(len(data)) if not is_local: if self._encryptor is not None: if self._encrypt_correct: @@ -667,8 +664,6 @@ def _on_remote_read(self, is_remote_sock): self.destroy() return if self._encryptor is not None: - self._server.server_transfer_dl += len(data) - self._update_activity(len(data)) if self._is_local: obfs_decode = self._obfs.client_decode(data) if obfs_decode[1]: @@ -683,6 +678,8 @@ def _on_remote_read(self, is_remote_sock): if self._encrypt_correct: data = self._protocol.server_pre_encrypt(data) data = self._encryptor.encrypt(data) + self._update_activity(len(data)) + self._server.server_transfer_dl += len(data) else: return try: From e61ab27e5982329f0d48ece22dc59910656ac0d6 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 9 Dec 2015 16:36:40 +0800 Subject: [PATCH 115/487] tls_simple bug fixed --- shadowsocks/obfsplugin/http_simple.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 2a7a1071..77946c45 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -194,6 +194,7 @@ def __init__(self, method): self.host = None self.port = 0 self.recv_buffer = b'' + self.send_buffer = b'' def client_encode(self, buf): if self.raw_trans_sent: @@ -276,6 +277,7 @@ def __init__(self, method): self.has_sent_header = False self.has_recv_header = False self.raw_trans_sent = False + self.send_buffer = b'' def client_encode(self, buf): if self.raw_trans_sent: From aa92a5940570143d8bcc694f778a014a1db484c8 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 10 Dec 2015 18:35:37 +0800 Subject: [PATCH 116/487] improve tls_simple --- shadowsocks/obfsplugin/http_simple.py | 33 ++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 77946c45..dd44eabd 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -307,8 +307,16 @@ def server_encode(self, buf): return buf self.has_sent_header = True # TODO - #server_hello = b'' - return b'\x16\x03\x01' + data = b"\x03\x03" + os.urandom(32) + data = b"\x02\x00" + struct.pack('>H', len(data)) + data + data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data + return data + + def decode_error_return(self, buf): + self.has_sent_header = True + if self.method == 'tls_simple': + return (b'E', False, False) + return (buf, True, False) def server_decode(self, buf): if self.has_recv_header: @@ -316,10 +324,23 @@ def server_decode(self, buf): self.has_recv_header = True if not match_begin(buf, b'\x16\x03\x01'): - self.has_sent_header = True - if self.method == 'tls_simple': - return (b'E', False, False) - return (buf, True, False) + return self.decode_error_return(buf); + buf = buf[3:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(buf); + buf = buf[2:] + if not match_begin(buf, b'\x01\x00'): #client hello + return self.decode_error_return(buf); + buf = buf[2:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(buf); + buf = buf[2:] + if not match_begin(buf, b'\x03\x03'): + return self.decode_error_return(buf); + buf = buf[2:] + verifyid = buf[:32] + buf = buf[32:] + sessionid = buf[:4] # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) From aff97d4ce8cba4dc8835037981512e22686fe0b3 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 14 Dec 2015 00:55:44 +0800 Subject: [PATCH 117/487] improve LRUCache add "tls1.0_session_auth" fix "auth_sha1" in local mode refine log --- shadowsocks/lru_cache.py | 75 +++---- shadowsocks/obfs.py | 3 +- shadowsocks/obfsplugin/auth.py | 16 +- shadowsocks/obfsplugin/http_simple.py | 78 -------- shadowsocks/obfsplugin/obfs_tls.py | 276 ++++++++++++++++++++++++++ shadowsocks/obfsplugin/verify.py | 6 +- shadowsocks/tcprelay.py | 18 +- 7 files changed, 342 insertions(+), 130 deletions(-) create mode 100644 shadowsocks/obfsplugin/obfs_tls.py diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index e67fdffe..3b3c2640 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -22,14 +22,14 @@ import logging import time - # this LRUCache is optimized for concurrency, not QPS # n: concurrency, keys stored in the cache # m: visits not timed out, proportional to QPS * timeout -# get & set is O(1), not O(n). thus we can support very large n -# TODO: if timeout or QPS is too large, then this cache is not very efficient, -# as sweep() causes long pause +# get & set is O(log(n)), not O(n). thus we can support very large n +# sweep is O((n - m)*log(n)) or O(1024*log(n)) at most, +# no metter how large the cache or timeout value is +SWEEP_MAX_ITEMS = 1024 class LRUCache(collections.MutableMapping): """This class is not thread safe""" @@ -38,32 +38,39 @@ def __init__(self, timeout=60, close_callback=None, *args, **kwargs): self.timeout = timeout self.close_callback = close_callback self._store = {} - self._time_to_keys = collections.defaultdict(list) + self._time_to_keys = collections.OrderedDict() self._keys_to_last_time = {} - self._last_visits = collections.deque() - self._closed_values = set() + self._visit_id = 0 self.update(dict(*args, **kwargs)) # use the free update to set keys def __getitem__(self, key): - # O(1) + # O(log(n)) t = time.time() - self._keys_to_last_time[key] = t - self._time_to_keys[t].append(key) - self._last_visits.append(t) + last_t, vid = self._keys_to_last_time[key] + self._keys_to_last_time[key] = (t, vid) + if last_t != t: + del self._time_to_keys[(last_t, vid)] + self._time_to_keys[(t, vid)] = key return self._store[key] def __setitem__(self, key, value): - # O(1) + # O(log(n)) t = time.time() - self._keys_to_last_time[key] = t + if key in self._keys_to_last_time: + last_t, vid = self._keys_to_last_time[key] + del self._time_to_keys[(last_t, vid)] + vid = self._visit_id + self._visit_id += 1 + self._keys_to_last_time[key] = (t, vid) self._store[key] = value - self._time_to_keys[t].append(key) - self._last_visits.append(t) + self._time_to_keys[(t, vid)] = key def __delitem__(self, key): - # O(1) + # O(log(n)) + last_t, vid = self._keys_to_last_time[key] del self._store[key] del self._keys_to_last_time[key] + del self._time_to_keys[(last_t, vid)] def __iter__(self): return iter(self._store) @@ -72,39 +79,33 @@ def __len__(self): return len(self._store) def sweep(self): - # O(m) + # O(n - m) now = time.time() c = 0 - while len(self._last_visits) > 0: - least = self._last_visits[0] - if now - least <= self.timeout: + while c < SWEEP_MAX_ITEMS: + if len(self._time_to_keys) == 0: + break + last_t, vid = iter(self._time_to_keys).next() + if now - last_t <= self.timeout: break + key = self._time_to_keys[(last_t, vid)] + value = self._store[key] if self.close_callback is not None: - for key in self._time_to_keys[least]: - if key in self._store: - if now - self._keys_to_last_time[key] > self.timeout: - value = self._store[key] - if value not in self._closed_values: - self.close_callback(value) - self._closed_values.add(value) - for key in self._time_to_keys[least]: - if key in self._store: - if now - self._keys_to_last_time[key] > self.timeout: - del self._store[key] - del self._keys_to_last_time[key] - c += 1 - self._last_visits.popleft() - del self._time_to_keys[least] + self.close_callback(value) + del self._store[key] + del self._keys_to_last_time[key] + del self._time_to_keys[(last_t, vid)] + c += 1 if c: - self._closed_values.clear() logging.debug('%d keys swept' % c) - + return c < SWEEP_MAX_ITEMS def test(): c = LRUCache(timeout=0.3) c['a'] = 1 assert c['a'] == 1 + c['a'] = 1 time.sleep(0.5) c.sweep() diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 1bfaf673..94a14f22 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -23,12 +23,13 @@ import logging from shadowsocks import common -from shadowsocks.obfsplugin import plain, http_simple, verify, auth +from shadowsocks.obfsplugin import plain, http_simple, obfs_tls, verify, auth method_supported = {} method_supported.update(plain.obfs_map) method_supported.update(http_simple.obfs_map) +method_supported.update(obfs_tls.obfs_map) method_supported.update(verify.obfs_map) method_supported.update(auth.obfs_map) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 2260bb41..0f639aa0 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -250,7 +250,7 @@ def client_post_decrypt(self, buf): if self.decrypt_packet_num == 0: return None else: - raise Exception('server_post_decrype data error') + raise Exception('client_post_decrypt data error') if length > len(self.recv_buf): break @@ -260,7 +260,7 @@ def client_post_decrypt(self, buf): if self.decrypt_packet_num == 0: return None else: - raise Exception('server_post_decrype data uncorrect CRC32') + raise Exception('client_post_decrypt data uncorrect CRC32') pos = common.ord(self.recv_buf[2]) + 2 out_buf += self.recv_buf[pos:length - 4] @@ -379,7 +379,7 @@ def pack_auth_data(self, buf): return b'' rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) data = common.chr(len(rnd_data) + 1) + rnd_data + buf - data = struct.pack('>H', len(data) + 10) + data + data = struct.pack('>H', len(data) + 16) + data crc = binascii.crc32(self.server_info.key) data = struct.pack(' len(self.recv_buf): break - if zlib.adler32(self.recv_buf[:length - 4]) != struct.unpack(' len(self.recv_buf): break - if zlib.adler32(self.recv_buf[:length - 4]) & 0xFFFFFFFF != struct.unpack('H', len(data)) + data - data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data - return data - if self.has_recv_header: - ret = self.send_buffer - self.send_buffer = b'' - self.raw_trans_sent = True - return ret - return b'' - - def client_decode(self, buf): - if self.has_recv_header: - return (buf, False) - self.has_recv_header = True - return (b'', True) - - def server_encode(self, buf): - if self.has_sent_header: - return buf - self.has_sent_header = True - # TODO - data = b"\x03\x03" + os.urandom(32) - data = b"\x02\x00" + struct.pack('>H', len(data)) + data - data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data - return data - - def decode_error_return(self, buf): - self.has_sent_header = True - if self.method == 'tls_simple': - return (b'E', False, False) - return (buf, True, False) - - def server_decode(self, buf): - if self.has_recv_header: - return (buf, True, False) - - self.has_recv_header = True - if not match_begin(buf, b'\x16\x03\x01'): - return self.decode_error_return(buf); - buf = buf[3:] - if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: - return self.decode_error_return(buf); - buf = buf[2:] - if not match_begin(buf, b'\x01\x00'): #client hello - return self.decode_error_return(buf); - buf = buf[2:] - if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: - return self.decode_error_return(buf); - buf = buf[2:] - if not match_begin(buf, b'\x03\x03'): - return self.decode_error_return(buf); - buf = buf[2:] - verifyid = buf[:32] - buf = buf[32:] - sessionid = buf[:4] - # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) - return (b'', False, True) - class random_head(plain.plain): def __init__(self, method): self.method = method diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py new file mode 100644 index 00000000..82f25d0c --- /dev/null +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python +# +# Copyright 2015-2015 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import hashlib +import logging +import binascii +import struct +import base64 +import time +import random +import hmac +import hashlib + +from shadowsocks import common +from shadowsocks.obfsplugin import plain +from shadowsocks.common import to_bytes, to_str, ord +from shadowsocks import lru_cache + +def create_tls_obfs(method): + return tls_simple(method) + +def create_tls_auth_obfs(method): + return tls_auth(method) + +obfs_map = { + 'tls_simple': (create_tls_obfs,), + 'tls_simple_compatible': (create_tls_obfs,), + 'tls1.0_session_auth': (create_tls_auth_obfs,), + 'tls1.0_session_auth_compatible': (create_tls_auth_obfs,), +} + +def match_begin(str1, str2): + if len(str1) >= len(str2): + if str1[:len(str2)] == str2: + return True + return False + +class tls_simple(plain.plain): + def __init__(self, method): + self.method = method + self.has_sent_header = False + self.has_recv_header = False + self.raw_trans_sent = False + self.send_buffer = b'' + self.tls_version = b'\x03\x01' + + def client_encode(self, buf): + if self.raw_trans_sent: + return buf + self.send_buffer += buf + if not self.has_sent_header: + self.has_sent_header = True + data = self.tls_version + os.urandom(32) + binascii.unhexlify(b"000016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") + data = b"\x01\x00" + struct.pack('>H', len(data)) + data + data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data + return data + if self.has_recv_header: + ret = self.send_buffer + self.send_buffer = b'' + self.raw_trans_sent = True + return ret + return b'' + + def client_decode(self, buf): + if self.has_recv_header: + return (buf, False) + self.has_recv_header = True + return (b'', True) + + def server_encode(self, buf): + if self.has_sent_header: + return buf + self.has_sent_header = True + # TODO + data = self.tls_version + os.urandom(32) + data = b"\x02\x00" + struct.pack('>H', len(data)) + data + data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data + return data + + def decode_error_return(self, buf): + self.has_sent_header = True + if self.method == 'tls_simple': + return (b'E', False, False) + return (buf, True, False) + + def server_decode(self, buf): + if self.has_recv_header: + return (buf, True, False) + + self.has_recv_header = True + if not match_begin(buf, b'\x16' + self.tls_version): + return self.decode_error_return(buf) + buf = buf[3:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(buf) + buf = buf[2:] + if not match_begin(buf, b'\x01\x00'): #client hello + return self.decode_error_return(buf) + buf = buf[2:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(buf) + buf = buf[2:] + if not match_begin(buf, self.tls_version): + return self.decode_error_return(buf) + buf = buf[2:] + verifyid = buf[:32] + buf = buf[32:] + sessionid_len = ord(buf[1]) + sessionid = buf[1:sessionid_len + 1] + buf = buf[sessionid_len+1:] + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (b'', False, True) + +class obfs_client_data(object): + def __init__(self, cid): + self.client_id = cid + self.auth_code = {} + +class obfs_auth_data(object): + def __init__(self): + self.client_data = lru_cache.LRUCache(60 * 5) + self.client_id = os.urandom(32) + self.startup_time = int(time.time() - 60 * 30) & 0xFFFFFFFF + +class tls_auth(plain.plain): + def __init__(self, method): + self.method = method + self.has_sent_header = False + self.has_recv_header = False + self.raw_trans_sent = False + self.raw_trans_recv = False + self.send_buffer = b'' + self.client_id = b'' + self.max_time_dif = 60 * 60 # time dif (second) setting + self.tls_version = b'\x03\x01' + + def init_data(self): + return obfs_auth_data() + + def pack_auth_data(self, client_id): + utc_time = int(time.time()) & 0xFFFFFFFF + data = struct.pack('>I', utc_time) + os.urandom(18) + data += hmac.new(self.server_info.key + client_id, data, hashlib.sha1).digest()[:10] + return data + + def client_encode(self, buf): + if self.raw_trans_sent: + return buf + self.send_buffer += buf + if not self.has_sent_header: + self.has_sent_header = True + data = self.tls_version + self.pack_auth_data(self.server_info.data.client_id) + b"\x20" + self.server_info.data.client_id + binascii.unhexlify(b"0016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") + data = b"\x01\x00" + struct.pack('>H', len(data)) + data + data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data + return data + if self.has_recv_header: + data = b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + "\x00\x01\x20" + os.urandom(22) #Finished + data += hmac.new(self.server_info.key + self.server_info.data.client_id, data, hashlib.sha1).digest()[:10] + ret = data + self.send_buffer + self.send_buffer = b'' + self.raw_trans_sent = True + return ret + return b'' + + def client_decode(self, buf): + if self.has_recv_header: + return (buf, False) + self.has_recv_header = True + return (b'', True) + + def server_encode(self, buf): + if self.has_sent_header: + return buf + self.has_sent_header = True + data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"0016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") + data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello + data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data + data += b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + "\x00\x01\x20" + os.urandom(22) #Finished + data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] + return data + + def decode_error_return(self, buf): + self.raw_trans_recv = True + if self.method == 'tls_simple': + return (b'E', False, False) + return (buf, True, False) + + def server_decode(self, buf): + if self.raw_trans_recv: + return (buf, True, False) + + if self.has_recv_header: + verify = buf + verify_len = 44 - 10 + if len(buf) < 44: + logging.error('server_decode data error') + return decode_error_return(b'') + if not match_begin(buf, b"\x14" + self.tls_version + "\x00\x01\x01"): #ChangeCipherSpec + logging.error('server_decode data error') + return decode_error_return(b'') + buf = buf[6:] + if not match_begin(buf, b"\x16" + self.tls_version + "\x00\x01\x20"): #Finished + logging.error('server_decode data error') + return decode_error_return(b'') + if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: + logging.error('server_decode data error') + return decode_error_return(b'') + if len(buf) < 38: + logging.error('server_decode data error') + return decode_error_return(b'') + buf = buf[38:] + self.raw_trans_recv = True + return (buf, True, False) + + self.has_recv_header = True + ogn_buf = buf + if not match_begin(buf, b'\x16' + self.tls_version): + return self.decode_error_return(ogn_buf) + buf = buf[3:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(ogn_buf) + buf = buf[2:] + if not match_begin(buf, b'\x01\x00'): #client hello + return self.decode_error_return(ogn_buf) + buf = buf[2:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(ogn_buf) + buf = buf[2:] + if not match_begin(buf, self.tls_version): + return self.decode_error_return(ogn_buf) + buf = buf[2:] + verifyid = buf[:32] + buf = buf[32:] + sessionid_len = ord(buf[0]) + if sessionid_len < 32: + logging.error("tls_auth wrong sessionid_len") + return self.decode_error_return(ogn_buf) + sessionid = buf[1:sessionid_len + 1] + buf = buf[sessionid_len+1:] + self.client_id = sessionid + sha1 = hmac.new(self.server_info.key + sessionid, verifyid[:22], hashlib.sha1).digest()[:10] + utc_time = struct.unpack('>I', verifyid[:4])[0] + time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time) + if time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ + or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: + logging.debug("tls_auth wrong time") + return self.decode_error_return(ogn_buf) + if sha1 != verifyid[22:]: + logging.debug("tls_auth wrong sha1") + return self.decode_error_return(ogn_buf) + if verifyid[4:22] in self.server_info.data.client_data: + logging.error("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) + return self.decode_error_return(ogn_buf) + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (b'', False, True) + diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 5c54c781..8b173455 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -127,7 +127,7 @@ def client_post_decrypt(self, buf): if self.decrypt_packet_num == 0: return None else: - raise Exception('server_post_decrype data error') + raise Exception('client_post_decrypt data error') if length > len(self.recv_buf): break @@ -137,7 +137,7 @@ def client_post_decrypt(self, buf): if self.decrypt_packet_num == 0: return None else: - raise Exception('server_post_decrype data uncorrect CRC32') + raise Exception('client_post_decrypt data uncorrect CRC32') pos = common.ord(self.recv_buf[2]) + 2 out_buf += self.recv_buf[pos:length - 4] @@ -224,7 +224,7 @@ def client_post_decrypt(self, buf): if self.decrypt_packet_num == 0: return None else: - raise Exception('server_post_decrype data error') + raise Exception('client_post_decrypt data error') if length > len(self.recv_buf): break diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 1f301aa6..2773d8b9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -598,7 +598,11 @@ def _on_local_read(self): if not is_local: if self._encryptor is not None: if self._encrypt_correct: - obfs_decode = self._obfs.server_decode(data) + try: + obfs_decode = self._obfs.server_decode(data) + except Exception as e: + shell.print_exception(e) + self.destroy() if obfs_decode[2]: self._write_to_sock(b'', self._local_sock) if obfs_decode[1]: @@ -665,7 +669,11 @@ def _on_remote_read(self, is_remote_sock): return if self._encryptor is not None: if self._is_local: - obfs_decode = self._obfs.client_decode(data) + try: + obfs_decode = self._obfs.client_decode(data) + except Exception as e: + shell.print_exception(e) + self.destroy() if obfs_decode[1]: send_back = self._obfs.client_encode(b'') self._write_to_sock(send_back, self._remote_sock) @@ -673,7 +681,11 @@ def _on_remote_read(self, is_remote_sock): iv_len = len(self._protocol.obfs.server_info.iv) self._protocol.obfs.server_info.recv_iv = obfs_decode[0][:iv_len] data = self._encryptor.decrypt(obfs_decode[0]) - data = self._protocol.client_post_decrypt(data) + try: + data = self._protocol.client_post_decrypt(data) + except Exception as e: + shell.print_exception(e) + self.destroy() else: if self._encrypt_correct: data = self._protocol.server_pre_encrypt(data) From 13579f1ca437f81175b638ebc2a49beec0ff33d6 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 14 Dec 2015 02:32:24 +0800 Subject: [PATCH 118/487] return after destory fix "tls1.0_session_auth" --- shadowsocks/obfsplugin/auth.py | 10 ++-------- shadowsocks/obfsplugin/obfs_tls.py | 4 +++- shadowsocks/obfsplugin/verify.py | 15 +++------------ shadowsocks/tcprelay.py | 3 +++ 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 0f639aa0..16ee20ae 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -247,20 +247,14 @@ def client_post_decrypt(self, buf): if length >= 8192 or length < 7: self.raw_trans = True self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('client_post_decrypt data error') + raise Exception('client_post_decrypt data error') if length > len(self.recv_buf): break if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: self.raw_trans = True self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('client_post_decrypt data uncorrect CRC32') + raise Exception('client_post_decrypt data uncorrect CRC32') pos = common.ord(self.recv_buf[2]) + 2 out_buf += self.recv_buf[pos:length - 4] diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 82f25d0c..3f98dd67 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -268,9 +268,11 @@ def server_decode(self, buf): if sha1 != verifyid[22:]: logging.debug("tls_auth wrong sha1") return self.decode_error_return(ogn_buf) - if verifyid[4:22] in self.server_info.data.client_data: + if self.server_info.data.client_data.get(verifyid[:22]): logging.error("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) return self.decode_error_return(ogn_buf) + self.server_info.data.client_data.sweep() + self.server_info.data.client_data[verifyid[:22]] = sessionid # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 8b173455..fada8d9e 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -124,20 +124,14 @@ def client_post_decrypt(self, buf): if length >= 8192 or length < 7: self.raw_trans = True self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('client_post_decrypt data error') + raise Exception('client_post_decrypt data error') if length > len(self.recv_buf): break if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: self.raw_trans = True self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('client_post_decrypt data uncorrect CRC32') + raise Exception('client_post_decrypt data uncorrect CRC32') pos = common.ord(self.recv_buf[2]) + 2 out_buf += self.recv_buf[pos:length - 4] @@ -221,10 +215,7 @@ def client_post_decrypt(self, buf): if length >= 32768 or length < 6: self.raw_trans = True self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('client_post_decrypt data error') + raise Exception('client_post_decrypt data error') if length > len(self.recv_buf): break diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 2773d8b9..ac9e8fbd 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -603,6 +603,7 @@ def _on_local_read(self): except Exception as e: shell.print_exception(e) self.destroy() + return if obfs_decode[2]: self._write_to_sock(b'', self._local_sock) if obfs_decode[1]: @@ -674,6 +675,7 @@ def _on_remote_read(self, is_remote_sock): except Exception as e: shell.print_exception(e) self.destroy() + return if obfs_decode[1]: send_back = self._obfs.client_encode(b'') self._write_to_sock(send_back, self._remote_sock) @@ -686,6 +688,7 @@ def _on_remote_read(self, is_remote_sock): except Exception as e: shell.print_exception(e) self.destroy() + return else: if self._encrypt_correct: data = self._protocol.server_pre_encrypt(data) From 443c4f35c21efd9fa532c05aafc3c9137decdf5f Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 14 Dec 2015 02:59:24 +0800 Subject: [PATCH 119/487] fix IPv6 detect --- shadowsocks/asyncdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 50c62bad..cd8550ac 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -81,7 +81,7 @@ def detect_ipv6_supprot(): if 'has_ipv6' in dir(socket): try: s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) - s.connect(('ipv6.google.com', 0)) + s.connect(('::1', 0)) print('IPv6 support') return True except: From 7300205add054203dfd9e0baa7a74fa0522305fa Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 14 Dec 2015 03:59:08 +0800 Subject: [PATCH 120/487] not check tls version --- shadowsocks/obfsplugin/obfs_tls.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 3f98dd67..30b65458 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -117,15 +117,6 @@ def server_decode(self, buf): buf = buf[2:] if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: return self.decode_error_return(buf) - buf = buf[2:] - if not match_begin(buf, self.tls_version): - return self.decode_error_return(buf) - buf = buf[2:] - verifyid = buf[:32] - buf = buf[32:] - sessionid_len = ord(buf[1]) - sessionid = buf[1:sessionid_len + 1] - buf = buf[sessionid_len+1:] # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) From 430b98c9827fbfb6f3cdc80c1588d77853cb22b5 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 14 Dec 2015 04:06:21 +0800 Subject: [PATCH 121/487] fix "tls1.0_session_auth_compatible" --- shadowsocks/obfsplugin/obfs_tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 30b65458..efb83f41 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -179,7 +179,7 @@ def client_decode(self, buf): return (b'', True) def server_encode(self, buf): - if self.has_sent_header: + if self.raw_trans_sent or self.has_sent_header: return buf self.has_sent_header = True data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"0016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") @@ -191,8 +191,9 @@ def server_encode(self, buf): return data def decode_error_return(self, buf): + self.raw_trans_sent = True self.raw_trans_recv = True - if self.method == 'tls_simple': + if self.method == 'tls1.0_session_auth': return (b'E', False, False) return (buf, True, False) From e889f953ff683d1d6af1e5f46015e8490a77be84 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 14 Dec 2015 14:55:39 +0800 Subject: [PATCH 122/487] UDP server_transfer_dl += len(post_data) --- shadowsocks/udprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index c6a69b70..80a30eb1 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -528,7 +528,6 @@ def _on_remote_read(self): if not data: self.destroy() return - self._server.server_transfer_dl += len(data) try: recv_data = data beg_pos = 0 @@ -547,6 +546,7 @@ def _on_remote_read(self): post_data = self._pack_post_data(CMD_POST, pack_id, data) addr = self.get_local_address() self._write_to_sock(post_data, self._local_sock, addr) + self._server.server_transfer_dl += len(post_data) if pack_id <= DOUBLE_SEND_BEG_IDS: post_data = self._pack_post_data(CMD_POST, pack_id, data) self._write_to_sock(post_data, self._local_sock, addr) From 8fb3cc33943a959e5e256bd5175e08773a0a89f4 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 17 Dec 2015 17:02:31 +0800 Subject: [PATCH 123/487] fix tls1.0 protocol --- shadowsocks/obfsplugin/obfs_tls.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index efb83f41..0daafe6b 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -206,20 +206,20 @@ def server_decode(self, buf): verify_len = 44 - 10 if len(buf) < 44: logging.error('server_decode data error') - return decode_error_return(b'') + return self.decode_error_return(b'') if not match_begin(buf, b"\x14" + self.tls_version + "\x00\x01\x01"): #ChangeCipherSpec logging.error('server_decode data error') - return decode_error_return(b'') + return self.decode_error_return(b'') buf = buf[6:] - if not match_begin(buf, b"\x16" + self.tls_version + "\x00\x01\x20"): #Finished + if not match_begin(buf, b"\x16" + self.tls_version + "\x00\x20"): #Finished logging.error('server_decode data error') - return decode_error_return(b'') + return self.decode_error_return(b'') if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: logging.error('server_decode data error') - return decode_error_return(b'') + return self.decode_error_return(b'') if len(buf) < 38: logging.error('server_decode data error') - return decode_error_return(b'') + return self.decode_error_return(b'') buf = buf[38:] self.raw_trans_recv = True return (buf, True, False) From 0c4b00a8af3ed9aa552d70151822e3b3e4b2cba3 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 19 Dec 2015 12:22:23 +0800 Subject: [PATCH 124/487] fix LRUCache --- shadowsocks/lru_cache.py | 51 ++++----- shadowsocks/ordereddict.py | 214 +++++++++++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+), 25 deletions(-) create mode 100644 shadowsocks/ordereddict.py diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 3b3c2640..648b5a3c 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -22,11 +22,22 @@ import logging import time +if __name__ == '__main__': + import os, sys, inspect + file_path = os.path.dirname(os.path.realpath(inspect.getfile(inspect.currentframe()))) + sys.path.insert(0, os.path.join(file_path, '../')) + +try: + from collections import OrderedDict + print("loaded collections.OrderedDict") +except: + from shadowsocks.ordereddict import OrderedDict + # this LRUCache is optimized for concurrency, not QPS # n: concurrency, keys stored in the cache # m: visits not timed out, proportional to QPS * timeout -# get & set is O(log(n)), not O(n). thus we can support very large n -# sweep is O((n - m)*log(n)) or O(1024*log(n)) at most, +# get & set is O(1), not O(n). thus we can support very large n +# sweep is O((n - m)) or O(1024) at most, # no metter how large the cache or timeout value is SWEEP_MAX_ITEMS = 1024 @@ -38,39 +49,30 @@ def __init__(self, timeout=60, close_callback=None, *args, **kwargs): self.timeout = timeout self.close_callback = close_callback self._store = {} - self._time_to_keys = collections.OrderedDict() - self._keys_to_last_time = {} - self._visit_id = 0 + self._keys_to_last_time = OrderedDict() self.update(dict(*args, **kwargs)) # use the free update to set keys def __getitem__(self, key): - # O(log(n)) + # O(1) t = time.time() - last_t, vid = self._keys_to_last_time[key] - self._keys_to_last_time[key] = (t, vid) - if last_t != t: - del self._time_to_keys[(last_t, vid)] - self._time_to_keys[(t, vid)] = key + last_t = self._keys_to_last_time[key] + del self._keys_to_last_time[key] + self._keys_to_last_time[key] = t return self._store[key] def __setitem__(self, key, value): - # O(log(n)) + # O(1) t = time.time() if key in self._keys_to_last_time: - last_t, vid = self._keys_to_last_time[key] - del self._time_to_keys[(last_t, vid)] - vid = self._visit_id - self._visit_id += 1 - self._keys_to_last_time[key] = (t, vid) + del self._keys_to_last_time[key] + self._keys_to_last_time[key] = t self._store[key] = value - self._time_to_keys[(t, vid)] = key def __delitem__(self, key): - # O(log(n)) - last_t, vid = self._keys_to_last_time[key] + # O(1) + last_t = self._keys_to_last_time[key] del self._store[key] del self._keys_to_last_time[key] - del self._time_to_keys[(last_t, vid)] def __iter__(self): return iter(self._store) @@ -83,18 +85,17 @@ def sweep(self): now = time.time() c = 0 while c < SWEEP_MAX_ITEMS: - if len(self._time_to_keys) == 0: + if len(self._keys_to_last_time) == 0: break - last_t, vid = iter(self._time_to_keys).next() + key = iter(self._keys_to_last_time).next() + last_t = self._keys_to_last_time[key] if now - last_t <= self.timeout: break - key = self._time_to_keys[(last_t, vid)] value = self._store[key] if self.close_callback is not None: self.close_callback(value) del self._store[key] del self._keys_to_last_time[key] - del self._time_to_keys[(last_t, vid)] c += 1 if c: logging.debug('%d keys swept' % c) diff --git a/shadowsocks/ordereddict.py b/shadowsocks/ordereddict.py new file mode 100644 index 00000000..e1918f5e --- /dev/null +++ b/shadowsocks/ordereddict.py @@ -0,0 +1,214 @@ +import collections + +################################################################################ +### OrderedDict +################################################################################ + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as regular dictionaries. + + # The internal self.__map dict maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(*args, **kwds): + '''Initialize an ordered dictionary. The signature is the same as + regular dictionaries, but keyword arguments are not recommended because + their insertion order is arbitrary. + + ''' + if not args: + raise TypeError("descriptor '__init__' of 'OrderedDict' object " + "needs an argument") + self = args[0] + args = args[1:] + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link at the end of the linked list, + # and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + return dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which gets + # removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, _ = self.__map.pop(key) + link_prev[1] = link_next # update link_prev[NEXT] + link_next[0] = link_prev # update link_next[PREV] + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + # Traverse the linked list in order. + root = self.__root + curr = root[1] # start at the first node + while curr is not root: + yield curr[2] # yield the curr[KEY] + curr = curr[1] # move to next node + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + # Traverse the linked list in reverse order. + root = self.__root + curr = root[0] # start at the last node + while curr is not root: + yield curr[2] # yield the curr[KEY] + curr = curr[0] # move to previous node + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + dict.clear(self) + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) pairs in od' + for k in self: + yield (k, self[k]) + + update = collections.MutableMapping.update + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise KeyError + is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + key = next(reversed(self) if last else iter(self)) + value = self.pop(key) + return key, value + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. + If not specified, the value defaults to None. + + ''' + self = cls() + for key in iterable: + self[key] = value + return self + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return dict.__eq__(self, other) and all(_imap(_eq, self, other)) + return dict.__eq__(self, other) + + def __ne__(self, other): + 'od.__ne__(y) <==> od!=y' + return not self == other + + # -- the following methods support python 3.x style dictionary views -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) + From 459b0c700450372da57cb1c41cab07929f454724 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 19 Dec 2015 12:46:47 +0800 Subject: [PATCH 125/487] add "redirect" in config.json --- config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config.json b/config.json index d4ffc7c3..eaaf50ca 100644 --- a/config.json +++ b/config.json @@ -11,6 +11,7 @@ "protocol_param": "", "obfs": "http_simple_compatible", "obfs_param": "", + "redirect": "", "dns_ipv6": false, "fast_open": false, "workers": 1 From 68a77ddc70b7f0864bf85e2feb47bc73795f1e99 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 19 Dec 2015 15:45:17 +0800 Subject: [PATCH 126/487] load 'user-config.json' first set default obfs plugin 'tls1.0_session_auth_compatible' --- config.json | 2 +- shadowsocks/shell.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index eaaf50ca..cbf648bf 100644 --- a/config.json +++ b/config.json @@ -9,7 +9,7 @@ "method": "aes-256-cfb", "protocol": "auth_sha1_compatible", "protocol_param": "", - "obfs": "http_simple_compatible", + "obfs": "tls1.0_session_auth_compatible", "obfs_param": "", "redirect": "", "dns_ipv6": false, diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 2d636bfd..342b8da5 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -62,8 +62,14 @@ def print_shadowsocks(): pass print('Shadowsocks %s' % version) - def find_config(): + config_path = 'user-config.json' + if os.path.exists(config_path): + return config_path + config_path = os.path.join(os.path.dirname(__file__), '../', 'user-config.json') + if os.path.exists(config_path): + return config_path + config_path = 'config.json' if os.path.exists(config_path): return config_path From 80604a9421606e2de36f25644748a6ba29f643c4 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 21 Dec 2015 18:03:52 +0800 Subject: [PATCH 127/487] fix auth_sha1 fix tls1.0_session_auth --- shadowsocks/obfsplugin/auth.py | 12 +++--------- shadowsocks/obfsplugin/obfs_tls.py | 19 +++++++++++++------ shadowsocks/tcprelay.py | 5 ++++- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 16ee20ae..14cff5db 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -374,7 +374,7 @@ def pack_auth_data(self, buf): rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) data = common.chr(len(rnd_data) + 1) + rnd_data + buf data = struct.pack('>H', len(data) + 16) + data - crc = binascii.crc32(self.server_info.key) + crc = binascii.crc32(self.server_info.key) & 0xFFFFFFFF data = struct.pack('= 8192 or length < 7: self.raw_trans = True self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return None - else: - raise Exception('client_post_decrypt data error') + raise Exception('client_post_decrypt data error') if length > len(self.recv_buf): break if struct.pack('H', len(data)) + data #server hello data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data data += b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + "\x00\x01\x20" + os.urandom(22) #Finished + data += b"\x16" + self.tls_version + "\x00\x20" + os.urandom(22) #Finished data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] return data @@ -203,8 +210,8 @@ def server_decode(self, buf): if self.has_recv_header: verify = buf - verify_len = 44 - 10 - if len(buf) < 44: + verify_len = 43 - 10 + if len(buf) < 43: logging.error('server_decode data error') return self.decode_error_return(b'') if not match_begin(buf, b"\x14" + self.tls_version + "\x00\x01\x01"): #ChangeCipherSpec @@ -217,10 +224,10 @@ def server_decode(self, buf): if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: logging.error('server_decode data error') return self.decode_error_return(b'') - if len(buf) < 38: + if len(buf) < 37: logging.error('server_decode data error') return self.decode_error_return(b'') - buf = buf[38:] + buf = buf[37:] self.raw_trans_recv = True return (buf, True, False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index ac9e8fbd..976f8b97 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -333,7 +333,10 @@ def _get_redirect_host(self, client_address, ogn_data): addr = struct.unpack('>I', address_bytes)[0] else: addr = 0 - host_post = common.to_str(host_list[((hash_code & 0xffffffff) + addr) % len(host_list)]) + if type(host_list) == list: + host_post = common.to_str(host_list[((hash_code & 0xffffffff) + addr) % len(host_list)]) + else: + host_post = host_list items = host_post.rsplit(':', 1) if len(items) > 1: try: From 202f0f614246eebcebd7cb100638839c98b47f33 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 21 Dec 2015 18:43:47 +0800 Subject: [PATCH 128/487] disconnect if wrong handshake package --- shadowsocks/obfsplugin/obfs_tls.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 732e7434..00be536e 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -176,12 +176,10 @@ def client_decode(self, buf): if self.has_recv_header: return (buf, False) if len(buf) < 11 + 32 + 1 + 32: - logging.error('client_decode data error') - return (b'', True) + raise Exception('client_decode data error') verify = buf[11:33] if hmac.new(self.server_info.key + self.server_info.data.client_id, verify, hashlib.sha1).digest()[:10] != buf[33:43]: - logging.error('client_decode data error') - return (b'', True) + raise Exception('client_decode data error') self.has_recv_header = True return (b'', True) @@ -212,21 +210,16 @@ def server_decode(self, buf): verify = buf verify_len = 43 - 10 if len(buf) < 43: - logging.error('server_decode data error') - return self.decode_error_return(b'') + raise Exception('server_decode data error') if not match_begin(buf, b"\x14" + self.tls_version + "\x00\x01\x01"): #ChangeCipherSpec - logging.error('server_decode data error') - return self.decode_error_return(b'') + raise Exception('server_decode data error') buf = buf[6:] if not match_begin(buf, b"\x16" + self.tls_version + "\x00\x20"): #Finished - logging.error('server_decode data error') - return self.decode_error_return(b'') + raise Exception('server_decode data error') if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: - logging.error('server_decode data error') - return self.decode_error_return(b'') + raise Exception('server_decode data error') if len(buf) < 37: - logging.error('server_decode data error') - return self.decode_error_return(b'') + raise Exception('server_decode data error') buf = buf[37:] self.raw_trans_recv = True return (buf, True, False) From 3dec2377bbc330a8dd096c8552b173391d2fe42b Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 22 Dec 2015 11:18:27 +0800 Subject: [PATCH 129/487] add "chacha20-ietf", libsodium version must be higher than 1.0.4, see https://github.com/jedisct1/libsodium/releases/ --- shadowsocks/crypto/sodium.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/shadowsocks/crypto/sodium.py b/shadowsocks/crypto/sodium.py index ae86fef3..fde5b168 100644 --- a/shadowsocks/crypto/sodium.py +++ b/shadowsocks/crypto/sodium.py @@ -52,6 +52,12 @@ def load_libsodium(): c_char_p, c_ulonglong, c_char_p) + libsodium.crypto_stream_chacha20_ietf_xor_ic.restype = c_int + libsodium.crypto_stream_chacha20_ietf_xor_ic.argtypes = (c_void_p, c_char_p, + c_ulonglong, + c_char_p, c_ulonglong, + c_char_p) + buf = create_string_buffer(buf_size) loaded = True @@ -68,6 +74,8 @@ def __init__(self, cipher_name, key, iv, op): self.cipher = libsodium.crypto_stream_salsa20_xor_ic elif cipher_name == 'chacha20': self.cipher = libsodium.crypto_stream_chacha20_xor_ic + elif cipher_name == 'chacha20-ietf': + self.cipher = libsodium.crypto_stream_chacha20_ietf_xor_ic else: raise Exception('Unknown cipher') # byte counter, not block counter @@ -97,6 +105,7 @@ def update(self, data): ciphers = { 'salsa20': (32, 8, SodiumCrypto), 'chacha20': (32, 8, SodiumCrypto), + 'chacha20-ietf': (32, 12, SodiumCrypto), } @@ -115,6 +124,14 @@ def test_chacha20(): util.run_cipher(cipher, decipher) +def test_chacha20_ietf(): + + cipher = SodiumCrypto('chacha20-ietf', b'k' * 32, b'i' * 16, 1) + decipher = SodiumCrypto('chacha20-ietf', b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + if __name__ == '__main__': + test_chacha20_ietf() test_chacha20() test_salsa20() From ddcf7f796f50aa766f22411112c4b5338547f822 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 22 Dec 2015 11:22:46 +0800 Subject: [PATCH 130/487] ignore error if not support "chacha20-ietf" --- shadowsocks/crypto/sodium.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shadowsocks/crypto/sodium.py b/shadowsocks/crypto/sodium.py index fde5b168..ad1f48fc 100644 --- a/shadowsocks/crypto/sodium.py +++ b/shadowsocks/crypto/sodium.py @@ -29,7 +29,7 @@ buf_size = 2048 -# for salsa20 and chacha20 +# for salsa20 and chacha20 and chacha20-ietf BLOCK_SIZE = 64 @@ -52,11 +52,14 @@ def load_libsodium(): c_char_p, c_ulonglong, c_char_p) - libsodium.crypto_stream_chacha20_ietf_xor_ic.restype = c_int - libsodium.crypto_stream_chacha20_ietf_xor_ic.argtypes = (c_void_p, c_char_p, + try: + libsodium.crypto_stream_chacha20_ietf_xor_ic.restype = c_int + libsodium.crypto_stream_chacha20_ietf_xor_ic.argtypes = (c_void_p, c_char_p, c_ulonglong, c_char_p, c_ulonglong, c_char_p) + except: + pass buf = create_string_buffer(buf_size) loaded = True From b151bf815fdb9df5e2d13bef825e816ffb9c2872 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 22 Dec 2015 19:39:39 +0800 Subject: [PATCH 131/487] fix "chacha20-ietf" --- shadowsocks/crypto/sodium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/sodium.py b/shadowsocks/crypto/sodium.py index ad1f48fc..c4b5c1c9 100644 --- a/shadowsocks/crypto/sodium.py +++ b/shadowsocks/crypto/sodium.py @@ -56,7 +56,7 @@ def load_libsodium(): libsodium.crypto_stream_chacha20_ietf_xor_ic.restype = c_int libsodium.crypto_stream_chacha20_ietf_xor_ic.argtypes = (c_void_p, c_char_p, c_ulonglong, - c_char_p, c_ulonglong, + c_char_p, c_ulong, c_char_p) except: pass From 42a05d22d63165fdbb904fab6a3d60f710ad9bfa Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 23 Dec 2015 18:13:27 +0800 Subject: [PATCH 132/487] support UDP protocol plugin --- shadowsocks/encrypt.py | 34 ++++++++++++++++++++++++++ shadowsocks/obfs.py | 12 ++++++++++ shadowsocks/obfsplugin/plain.py | 12 ++++++++++ shadowsocks/obfsplugin/verify.py | 16 +++++++++++++ shadowsocks/udprelay.py | 41 ++++++++++++++++++++++++++------ 5 files changed, 108 insertions(+), 7 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 24943957..c3fc0690 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -159,6 +159,40 @@ def encrypt_all(password, method, op, data): result.append(cipher.update(data)) return b''.join(result) +def encrypt_key(password, method): + method = method.lower() + (key_len, iv_len, m) = method_supported[method] + if key_len > 0: + key, _ = EVP_BytesToKey(password, key_len, iv_len) + else: + key = password + return key + +def encrypt_iv_len(method): + method = method.lower() + (key_len, iv_len, m) = method_supported[method] + return iv_len + +def encrypt_new_iv(method): + method = method.lower() + (key_len, iv_len, m) = method_supported[method] + return random_string(iv_len) + +def encrypt_all_iv(key, method, op, data, ref_iv): + result = [] + method = method.lower() + (key_len, iv_len, m) = method_supported[method] + if op: + iv = ref_iv[0] + result.append(iv) + else: + iv = data[:iv_len] + data = data[iv_len:] + ref_iv[0] = iv + cipher = m(method, key, iv, op) + result.append(cipher.update(data)) + return b''.join(result) + CIPHERS_TO_TEST = [ 'aes-128-cfb', diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 94a14f22..6adb8550 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -85,6 +85,18 @@ def server_decode(self, buf): def server_post_decrypt(self, buf): return self.obfs.server_post_decrypt(buf) + def client_udp_pre_encrypt(self, buf): + return self.obfs.client_udp_pre_encrypt(buf) + + def client_udp_post_decrypt(self, buf): + return self.obfs.client_udp_post_decrypt(buf) + + def server_udp_pre_encrypt(self, buf): + return self.obfs.server_udp_pre_encrypt(buf) + + def server_udp_post_decrypt(self, buf): + return self.obfs.server_udp_post_decrypt(buf) + def dispose(self): self.obfs.dispose() del self.obfs diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index 5bba57ca..8bb0000b 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -69,6 +69,18 @@ def server_decode(self, buf): def server_post_decrypt(self, buf): return buf + def client_udp_pre_encrypt(self, buf): + return buf + + def client_udp_post_decrypt(self, buf): + return buf + + def server_udp_pre_encrypt(self, buf): + return buf + + def server_udp_post_decrypt(self, buf): + return buf + def dispose(self): pass diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index fada8d9e..639ba5cb 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -342,3 +342,19 @@ def server_post_decrypt(self, buf): return out_buf + def client_udp_pre_encrypt(self, buf): + ret = self.pack_auth_data(buf) + return chr(ord(buf[0]) | 0x10) + buf[1:] + + def server_udp_post_decrypt(self, buf): + if buf and ((ord(buf[0]) & 0x10) == 0x10): + if len(buf) <= 11: + return b'E' + sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, buf[:-10], hashlib.sha1).digest()[:10] + if sha1data != buf[-10:]: + logging.error('server_udp_post_decrypt data uncorrect auth HMAC-SHA1') + return b'E' + return to_bytes(chr(ord(buf[0]) & 0xEF)) + buf[1:-10] + else: + return buf + diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 80a30eb1..d5cab1bd 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -71,7 +71,7 @@ import binascii import traceback -from shadowsocks import encrypt, eventloop, lru_cache, common, shell +from shadowsocks import encrypt, obfs, eventloop, lru_cache, common, shell from shadowsocks.common import pre_parse_header, parse_header, pack_addr # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time @@ -890,6 +890,20 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self.server_transfer_ul = 0 self.server_transfer_dl = 0 + self.protocol_data = obfs.obfs(config['protocol']).init_data() + self._protocol = obfs.obfs(config['protocol']) + server_info = obfs.server_info(self.protocol_data) + server_info.host = self._listen_addr + server_info.port = self._listen_port + server_info.protocol_param = config['protocol_param'] + server_info.obfs_param = '' + server_info.iv = b'' + server_info.recv_iv = b'' + server_info.key = encrypt.encrypt_key(self._password, self._method) + server_info.head_len = 30 + server_info.tcp_mss = 1440 + self._protocol.set_server_info(server_info) + self._sockets = set() self._fd_to_handlers = {} self._reqid_to_hd = {} @@ -991,11 +1005,14 @@ def _handle_server(self): else: data = data[3:] else: - data = encrypt.encrypt_all(self._password, self._method, 0, data) + ref_iv = [0] + data = encrypt.encrypt_all_iv(self._protocol.obfs.server_info.key, self._method, 0, data, ref_iv) # decrypt data if not data: logging.debug('UDP handle_server: data is empty after decrypt') return + self._protocol.obfs.server_info.recv_iv = ref_iv[0] + data = self._protocol.server_udp_post_decrypt(data) #logging.info("UDP data %s" % (binascii.hexlify(data),)) if not self._is_local: @@ -1121,7 +1138,11 @@ def _handle_server(self): r_addr[0], r_addr[1])) if self._is_local: - data = encrypt.encrypt_all(self._password, self._method, 1, data) + ref_iv = [encrypt.encrypt_new_iv(self._method)] + self._protocol.obfs.server_info.iv = ref_iv[0] + data = self._protocol.client_udp_pre_encrypt(data) + logging.info("%s" % (binascii.hexlify(data),)) + data = encrypt.encrypt_all_iv(self._protocol.obfs.server_info.key, self._method, 1, data, ref_iv) if not data: return else: @@ -1151,15 +1172,21 @@ def _handle_client(self, sock): # drop return data = pack_addr(r_addr[0]) + struct.pack('>H', r_addr[1]) + data - response = encrypt.encrypt_all(self._password, self._method, 1, - data) + ref_iv = [encrypt.encrypt_new_iv(self._method)] + self._protocol.obfs.server_info.iv = ref_iv[0] + data = self._protocol.server_udp_pre_encrypt(data) + response = encrypt.encrypt_all_iv(self._protocol.obfs.server_info.key, self._method, 1, + data, ref_iv) if not response: return else: - data = encrypt.encrypt_all(self._password, self._method, 0, - data) + ref_iv = [0] + data = encrypt.encrypt_all_iv(self._protocol.obfs.server_info.key, self._method, 0, + data, ref_iv) if not data: return + self._protocol.obfs.server_info.recv_iv = ref_iv[0] + data = self._protocol.client_udp_post_decrypt(data) header_result = parse_header(data) if header_result is None: return From 519c84719fb9017de504db677b9c9cb9bb586654 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 23 Dec 2015 18:19:57 +0800 Subject: [PATCH 133/487] add "protocol_param" --- shadowsocks/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index c0b6ed8e..f672de91 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -74,9 +74,12 @@ def main(): if type(password_obfs) == list: password = password_obfs[0] obfs = password_obfs[1] + if len(password_obfs) >= 2: + protocol = password_obfs[2] elif type(password_obfs) == dict: password = password_obfs.get('password', 'm') protocol = password_obfs.get('protocol', 'origin') + protocol_param = password_obfs.get('protocol_param', '') obfs = password_obfs.get('obfs', 'plain') obfs_param = password_obfs.get('obfs_param', '') else: From e56c5e4b90f0f72998a299072424af5ac22b009a Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 25 Dec 2015 11:45:19 +0800 Subject: [PATCH 134/487] fix index --- shadowsocks/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index f672de91..6ec63c0d 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -74,7 +74,7 @@ def main(): if type(password_obfs) == list: password = password_obfs[0] obfs = password_obfs[1] - if len(password_obfs) >= 2: + if len(password_obfs) > 2: protocol = password_obfs[2] elif type(password_obfs) == dict: password = password_obfs.get('password', 'm') From 876bbbe5d83207bf7cb841c6295b03d84def66df Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 28 Dec 2015 02:12:33 +0800 Subject: [PATCH 135/487] allow default settings --- shadowsocks/server.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 6ec63c0d..ddc206c4 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -40,10 +40,7 @@ def main(): daemon.daemon_exec(config) if config['port_password']: - if config['password']: - logging.warn('warning: port_password should not be used with ' - 'server_port and password. server_port and password ' - 'will be ignored') + pass else: config['port_password'] = {} server_port = config['server_port'] @@ -65,6 +62,7 @@ def main(): udp_servers = [] dns_resolver = asyncdns.DNSResolver() port_password = config['port_password'] + config_password = config.get('password', 'm') del config['port_password'] for port, password_obfs in port_password.items(): protocol = config.get("protocol", 'origin') @@ -77,11 +75,11 @@ def main(): if len(password_obfs) > 2: protocol = password_obfs[2] elif type(password_obfs) == dict: - password = password_obfs.get('password', 'm') - protocol = password_obfs.get('protocol', 'origin') - protocol_param = password_obfs.get('protocol_param', '') - obfs = password_obfs.get('obfs', 'plain') - obfs_param = password_obfs.get('obfs_param', '') + password = password_obfs.get('password', config_password) + protocol = password_obfs.get('protocol', protocol) + protocol_param = password_obfs.get('protocol_param', protocol_param) + obfs = password_obfs.get('obfs', obfs) + obfs_param = password_obfs.get('obfs_param', obfs_param) else: password = password_obfs a_config = config.copy() From 90ae1fdc4034d0394082a056b0fbc002b86aaed8 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 28 Dec 2015 15:05:47 +0800 Subject: [PATCH 136/487] refine manyuser thread start & stop --- db_transfer.py | 44 ++++++++++++++++++++++++++++++++------------ server.py | 14 +++++++++++--- server_pool.py | 3 +++ 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index caf60f3b..fb8f5d96 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -15,7 +15,9 @@ class DbTransfer(object): instance = None def __init__(self): + import threading self.last_get_transfer = {} + self.event = threading.Event() @staticmethod def get_instance(): @@ -148,6 +150,15 @@ def del_server_out_of_bound_safe(last_rows, rows): logging.info('db stop server at port [%s] reason: port not exist' % (row['port'])) ServerPool.get_instance().cb_del_server(row['port']) + @staticmethod + def del_servers(): + for port in ServerPool.get_instance().tcp_servers_pool.keys(): + if ServerPool.get_instance().server_is_run(port) > 0: + ServerPool.get_instance().cb_del_server(port) + for port in ServerPool.get_instance().tcp_ipv6_servers_pool.keys(): + if ServerPool.get_instance().server_is_run(port) > 0: + ServerPool.get_instance().cb_del_server(port) + @staticmethod def thread_db(): import socket @@ -155,16 +166,25 @@ def thread_db(): timeout = 60 socket.setdefaulttimeout(timeout) last_rows = [] - while True: - try: - DbTransfer.get_instance().push_db_all_user() - rows = DbTransfer.get_instance().pull_db_all_user() - DbTransfer.del_server_out_of_bound_safe(last_rows, rows) - last_rows = rows - except Exception as e: - trace = traceback.format_exc() - logging.error(trace) - #logging.warn('db thread except:%s' % e) - finally: - time.sleep(15) + try: + while True: + try: + DbTransfer.get_instance().push_db_all_user() + rows = DbTransfer.get_instance().pull_db_all_user() + DbTransfer.del_server_out_of_bound_safe(last_rows, rows) + last_rows = rows + except Exception as e: + trace = traceback.format_exc() + logging.error(trace) + #logging.warn('db thread except:%s' % e) + if DbTransfer.get_instance().event.wait(15): + break + except KeyboardInterrupt as e: + pass + DbTransfer.del_servers() + ServerPool.get_instance().stop() + + @staticmethod + def thread_db_stop(): + DbTransfer.get_instance().event.set() diff --git a/server.py b/server.py index ac32a7c6..a28f026d 100644 --- a/server.py +++ b/server.py @@ -39,15 +39,23 @@ def __init__(self): def run(self): db_transfer.DbTransfer.thread_db() + def stop(self): + db_transfer.DbTransfer.thread_db_stop() + def main(): shell.check_python() - if True: + if False: db_transfer.DbTransfer.thread_db() else: thread = MainThread() thread.start() - while True: - time.sleep(99999) + try: + while True: + time.sleep(99999) + except (KeyboardInterrupt, IOError, OSError) as e: + import traceback + traceback.print_exc() + thread.stop() if __name__ == '__main__': main() diff --git a/server_pool.py b/server_pool.py index 002ddfe1..6c751c16 100644 --- a/server_pool.py +++ b/server_pool.py @@ -72,6 +72,9 @@ def get_instance(): ServerPool.instance = ServerPool() return ServerPool.instance + def stop(self): + self.loop.stop() + @staticmethod def _loop(loop, dns_resolver, mgr): try: From 6576359c1cd9be622da961bf21301be481c84240 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 28 Dec 2015 19:35:06 +0800 Subject: [PATCH 137/487] fix "ord" --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 976f8b97..f73f8421 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -401,13 +401,13 @@ def _handle_stage_connecting(self, data): def _get_head_size(self, buf, def_value): if len(buf) < 2: return def_value - head_type = ord(buf[0]) & 0xF + head_type = common.ord(buf[0]) & 0xF if head_type == 1: return 7 if head_type == 4: return 19 if head_type == 3: - return 4 + ord(buf[1]) + return 4 + common.ord(buf[1]) return def_value def _handle_stage_addr(self, ogn_data, data): From e76939981a379c20351048a115311db9e056a6e8 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 28 Dec 2015 20:48:14 +0800 Subject: [PATCH 138/487] fix "chr" --- shadowsocks/obfsplugin/http_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index e4dcdbea..00a3dce1 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -29,7 +29,7 @@ from shadowsocks import common from shadowsocks.obfsplugin import plain -from shadowsocks.common import to_bytes, to_str, ord +from shadowsocks.common import to_bytes, to_str, ord, chr def create_http_obfs(method): return http_simple(method) @@ -79,7 +79,7 @@ def __init__(self, method): def encode_head(self, buf): ret = b'' for ch in buf: - ret += '%' + binascii.hexlify(ch) + ret += '%' + binascii.hexlify(chr(ch)) return ret def client_encode(self, buf): From 81b059759147545a06a8167a6b7b240ef9246b3c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 28 Dec 2015 21:32:02 +0800 Subject: [PATCH 139/487] fix "bytes" in "encode_head" --- shadowsocks/obfsplugin/http_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 00a3dce1..e1e359bb 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -79,7 +79,7 @@ def __init__(self, method): def encode_head(self, buf): ret = b'' for ch in buf: - ret += '%' + binascii.hexlify(chr(ch)) + ret += b'%' + binascii.hexlify(chr(ch)) return ret def client_encode(self, buf): From a49b21ee2c1c9bfe9f0f5b58e418d366463d8807 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 29 Dec 2015 00:37:36 +0800 Subject: [PATCH 140/487] fix "bytes" in "encode_head" 2 --- shadowsocks/obfsplugin/http_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index e1e359bb..f0d563f3 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -79,7 +79,7 @@ def __init__(self, method): def encode_head(self, buf): ret = b'' for ch in buf: - ret += b'%' + binascii.hexlify(chr(ch)) + ret += b'%' + to_bytes(binascii.hexlify(chr(ch))) return ret def client_encode(self, buf): From 8b7db6d9962c05cc46554ae8945661c1216fd491 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 29 Dec 2015 01:31:52 +0800 Subject: [PATCH 141/487] fix host name type --- shadowsocks/obfsplugin/http_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index f0d563f3..02b88979 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -94,9 +94,9 @@ def client_encode(self, buf): buf = buf[headlen:] port = b'' if self.server_info.port != 80: - port = b':' + common.to_bytes(str(self.server_info.port)) + port = b':' + to_bytes(str(self.server_info.port)) http_head = b"GET /" + self.encode_head(headdata) + b" HTTP/1.1\r\n" - http_head += b"Host: " + (self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" + http_head += b"Host: " + to_bytes(self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\nDNT: 1\r\nConnection: keep-alive\r\n\r\n" self.has_sent_header = True From f5cc946796bc27e586c3688528bce711a8d3ed11 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 10 Jan 2016 00:19:12 +0800 Subject: [PATCH 142/487] fix "http_simple" in py2/py3 --- shadowsocks/obfsplugin/http_simple.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 02b88979..2643c25f 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -77,10 +77,8 @@ def __init__(self, method): b"Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"] def encode_head(self, buf): - ret = b'' - for ch in buf: - ret += b'%' + to_bytes(binascii.hexlify(chr(ch))) - return ret + hexstr = binascii.hexlify(buf) + return b''.join([b"%" + pair for pair in [hexstr[i:i+2] for i in range(0, len(hexstr), 2)]]) def client_encode(self, buf): if self.has_sent_header: From 187e266b26d1513ce12c201821dcf1bf790efc39 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 18 Jan 2016 18:07:14 +0800 Subject: [PATCH 143/487] refine "http_simple" in py3.4 --- shadowsocks/obfsplugin/http_simple.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 2643c25f..7b7fe88f 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -78,7 +78,10 @@ def __init__(self, method): def encode_head(self, buf): hexstr = binascii.hexlify(buf) - return b''.join([b"%" + pair for pair in [hexstr[i:i+2] for i in range(0, len(hexstr), 2)]]) + chs = [] + for i in range(0, len(hexstr), 2): + chs.append(b"%" + hexstr[i:i+2]) + return b''.join(chs) def client_encode(self, buf): if self.has_sent_header: From 397990534a4d1d5c8823884bd73b4a028c5e1660 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jan 2016 20:37:40 +0800 Subject: [PATCH 144/487] add plugin "auth_sha1_v2" --- shadowsocks/lru_cache.py | 4 + shadowsocks/obfsplugin/auth.py | 248 ++++++++++++++++++++++++++++++++- 2 files changed, 251 insertions(+), 1 deletion(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 648b5a3c..e50d0b51 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -80,6 +80,10 @@ def __iter__(self): def __len__(self): return len(self._store) + def first(self): + if len(self._keys_to_last_time) > 0: + return iter(self._keys_to_last_time).next() + def sweep(self): # O(n - m) now = time.time() diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 14cff5db..16bfc160 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -32,7 +32,7 @@ import hashlib import shadowsocks -from shadowsocks import common +from shadowsocks import common, lru_cache from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord, chr @@ -42,10 +42,15 @@ def create_auth_simple(method): def create_auth_sha1(method): return auth_sha1(method) +def create_auth_sha1_v2(method): + return auth_sha1_v2(method) + obfs_map = { 'auth_simple': (create_auth_simple,), 'auth_sha1': (create_auth_sha1,), 'auth_sha1_compatible': (create_auth_sha1,), + 'auth_sha1_v2': (create_auth_sha1_v2,), + 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), } def match_begin(str1, str2): @@ -526,3 +531,244 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return out_buf +class obfs_auth_v2_data(object): + def __init__(self): + self.client_id = lru_cache.LRUCache() + self.local_client_id = b'' + self.connection_id = 0 + self.set_max_client(64) # max active client count + + def update(self, client_id, connection_id): + if client_id in self.client_id: + self.client_id[client_id].update() + + def set_max_client(self, max_client): + self.max_client = max_client + self.max_buffer = max(self.max_client * 2, 1024) + + def insert(self, client_id, connection_id): + if self.client_id.get(client_id, None) is None or not self.client_id[client_id].enable: + if self.client_id.first() is None or len(self.client_id) < self.max_client: + if client_id not in self.client_id: + #TODO: check + self.client_id[client_id] = client_queue(connection_id) + else: + self.client_id[client_id].re_enable(connection_id) + return self.client_id[client_id].insert(connection_id) + + if not self.client_id[self.client_id.first()].is_active(): + del self.client_id[self.client_id.first()] + if client_id not in self.client_id: + #TODO: check + self.client_id[client_id] = client_queue(connection_id) + else: + self.client_id[client_id].re_enable(connection_id) + return self.client_id[client_id].insert(connection_id) + + logging.warn('auth_sha1_v2: no inactive client') + return False + else: + return self.client_id[client_id].insert(connection_id) + +class auth_sha1_v2(verify_base): + def __init__(self, method): + super(auth_sha1_v2, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + + def init_data(self): + return obfs_auth_v2_data() + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + + def rnd_data(self, buf_size): + if buf_size > 1500: + return b'\x01' + + if buf_size > 400: + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) + return common.chr(len(rnd_data) + 1) + rnd_data + + rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 1024) + return common.chr(255) + struct.pack('>H', len(rnd_data) + 3) + rnd_data + + def pack_data(self, buf): + if len(buf) == 0: + return b'' + data = self.rnd_data(len(buf)) + buf + data = struct.pack('>H', len(data) + 6) + data + adler32 = zlib.adler32(data) & 0xFFFFFFFF + data += struct.pack('H', len(data) + 16) + data + crc = binascii.crc32(self.server_info.key) & 0xFFFFFFFF + data = struct.pack(' 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(8) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data error') + if length > len(self.recv_buf): + break + + if struct.pack('H', self.recv_buf[3:5])[0] + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf + + def server_pre_encrypt(self, buf): + if self.raw_trans: + return buf + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + if not self.has_recv_header: + if len(self.recv_buf) < 4: + return b'' + crc = struct.pack('H', self.recv_buf[4:6])[0] + if length > len(self.recv_buf): + return b'' + sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] + if sha1data != self.recv_buf[length - 10:length]: + logging.error('auth_sha1_v2 data uncorrect auth HMAC-SHA1') + return b'E' + pos = common.ord(self.recv_buf[6]) + if pos < 255: + pos += 6 + else: + pos = struct.unpack('>H', self.recv_buf[7:9])[0] + 6 + out_buf = self.recv_buf[pos:length - 10] + if len(out_buf) < 8: + self.raw_trans = True + self.recv_buf = b'' + logging.info('auth_sha1_v2: too short') + return b'E' + client_id = struct.unpack(' 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + logging.info('auth_sha1_v2: over size') + return b'E' + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if struct.pack('H', self.recv_buf[3:5])[0] + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.server_info.data.update(self.client_id, self.connection_id) + self.decrypt_packet_num += 1 + return out_buf + From 27d2214337ab94644c7185e9bf32113bd192909c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 20 Jan 2016 11:53:03 +0800 Subject: [PATCH 145/487] "auth_sha1_v2" add salt --- shadowsocks/obfsplugin/auth.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 16bfc160..c6ae0812 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -581,6 +581,7 @@ def __init__(self, method): self.has_recv_header = False self.client_id = 0 self.connection_id = 0 + self.salt = b"auth_sha1_v2" def init_data(self): return obfs_auth_v2_data() @@ -618,7 +619,7 @@ def pack_auth_data(self, buf): return b'' data = self.rnd_data(len(buf)) + buf data = struct.pack('>H', len(data) + 16) + data - crc = binascii.crc32(self.server_info.key) & 0xFFFFFFFF + crc = binascii.crc32(self.salt + self.server_info.key) & 0xFFFFFFFF data = struct.pack(' Date: Sat, 23 Jan 2016 22:14:45 +0800 Subject: [PATCH 146/487] better speed --- shadowsocks/obfsplugin/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index c6ae0812..1217cc46 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -595,7 +595,7 @@ def set_server_info(self, server_info): self.server_info.data.set_max_client(max_client) def rnd_data(self, buf_size): - if buf_size > 1500: + if buf_size > 1300: return b'\x01' if buf_size > 400: From 16eb3e3448ca3d5ae225fc738068cc140c05d4c5 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 15 Feb 2016 16:19:29 +0800 Subject: [PATCH 147/487] optimize sql update --- db_transfer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db_transfer.py b/db_transfer.py index fb8f5d96..6e9ad17e 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -57,6 +57,8 @@ def push_db_all_user(self): query_sub_in = None last_time = time.time() for id in dt_transfer.keys(): + if dt_transfer[id][0] == 0 and dt_transfer[id][1] == 0: + continue query_sub_when += ' WHEN %s THEN u+%s' % (id, dt_transfer[id][0]) query_sub_when2 += ' WHEN %s THEN d+%s' % (id, dt_transfer[id][1]) if query_sub_in is not None: From 41e92e277e6eb9846d1ac21d95b8fc16099cdcca Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 17 Feb 2016 21:17:38 +0800 Subject: [PATCH 148/487] fix iter.next ? --- shadowsocks/lru_cache.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index e50d0b51..32bab665 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -82,7 +82,8 @@ def __len__(self): def first(self): if len(self._keys_to_last_time) > 0: - return iter(self._keys_to_last_time).next() + for key in self._keys_to_last_time: + return key def sweep(self): # O(n - m) @@ -91,7 +92,8 @@ def sweep(self): while c < SWEEP_MAX_ITEMS: if len(self._keys_to_last_time) == 0: break - key = iter(self._keys_to_last_time).next() + for key in self._keys_to_last_time: + break last_t = self._keys_to_last_time[key] if now - last_t <= self.timeout: break From 77abc7d1bc6b7b9914dbc5a3b69c5aa2d0e02f08 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 31 Mar 2016 01:08:30 +0800 Subject: [PATCH 149/487] support big UDP data --- shadowsocks/tcprelay.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f73f8421..97d8594a 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -232,11 +232,15 @@ def _write_to_sock(self, data, sock): #logging.info('UDP over TCP sendto %d %s' % (len(data), binascii.hexlify(data))) while len(self._udp_data_send_buffer) > 6: length = struct.unpack('>H', self._udp_data_send_buffer[:2])[0] + if length >= 0xff00: + length = struct.unpack('>H', self._udp_data_send_buffer[1:3])[0] + 0xff00 if length > len(self._udp_data_send_buffer): break data = self._udp_data_send_buffer[:length] + if length >= 0xff00: + data = data[1:] self._udp_data_send_buffer = self._udp_data_send_buffer[length:] frag = common.ord(data[2]) @@ -660,7 +664,11 @@ def _on_remote_read(self, is_remote_sock): except Exception as e: ip = socket.inet_pton(socket.AF_INET6, addr[0]) data = b'\x00\x04' + ip + port + data - data = struct.pack('>H', len(data) + 2) + data + size = len(data) + 2 + if size >= 0xff00: + data = common.chr(0xff) + struct.pack('>H', size - 0xff00 + 1) + data + else: + data = struct.pack('>H', size) + data #logging.info('UDP over TCP recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) else: data = self._remote_sock.recv(BUF_SIZE) From cb86c34c3b43bd9396979c0694e7f1089acb9787 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 31 Mar 2016 02:24:10 +0800 Subject: [PATCH 150/487] UDP recv size --- shadowsocks/tcprelay.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 97d8594a..c07b28cb 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -89,7 +89,7 @@ WAIT_STATUS_READWRITING = WAIT_STATUS_READING | WAIT_STATUS_WRITING BUF_SIZE = 32 * 1024 - +UDP_MAX_BUF_SIZE = 65536 class TCPRelayHandler(object): def __init__(self, server, fd_to_handlers, loop, local_sock, config, @@ -654,9 +654,9 @@ def _on_remote_read(self, is_remote_sock): try: if self._remote_udp: if is_remote_sock: - data, addr = self._remote_sock.recvfrom(BUF_SIZE) + data, addr = self._remote_sock.recvfrom(UDP_MAX_BUF_SIZE) else: - data, addr = self._remote_sock_v6.recvfrom(BUF_SIZE) + data, addr = self._remote_sock_v6.recvfrom(UDP_MAX_BUF_SIZE) port = struct.pack('>H', addr[1]) try: ip = socket.inet_aton(addr[0]) From 14edb58281205292423d80f7efd0cce850882681 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 6 Apr 2016 14:42:27 +0800 Subject: [PATCH 151/487] DNS VALID_HOSTNAME add underline --- shadowsocks/asyncdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index cd8550ac..ebd5e9e9 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -35,7 +35,7 @@ CACHE_SWEEP_INTERVAL = 30 -VALID_HOSTNAME = re.compile(br"(?!-)[A-Z\d-]{1,63}(? Date: Mon, 11 Apr 2016 09:37:42 +0800 Subject: [PATCH 152/487] fix loading obfs plugin on py3 --- shadowsocks/obfs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 6adb8550..f34c7c0f 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -39,6 +39,7 @@ def __init__(self, data): class obfs(object): def __init__(self, method): + method = common.to_str(method) self.method = method self._method_info = self.get_method_info(method) if self._method_info: From 95bc6e4a2fb16611cacec481ab582f628a0d810b Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 17 Apr 2016 11:32:49 +0800 Subject: [PATCH 153/487] tcp resend data bug fix add tls1.2_ticket_auth --- config.json | 2 +- shadowsocks/obfsplugin/http_simple.py | 2 +- shadowsocks/obfsplugin/obfs_tls.py | 199 ++++++++++++++++++++++++++ shadowsocks/tcprelay.py | 7 +- 4 files changed, 204 insertions(+), 6 deletions(-) diff --git a/config.json b/config.json index cbf648bf..eaaf50ca 100644 --- a/config.json +++ b/config.json @@ -9,7 +9,7 @@ "method": "aes-256-cfb", "protocol": "auth_sha1_compatible", "protocol_param": "", - "obfs": "tls1.0_session_auth_compatible", + "obfs": "http_simple_compatible", "obfs_param": "", "redirect": "", "dns_ipv6": false, diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 7b7fe88f..4fd5f450 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -119,7 +119,7 @@ def server_encode(self, buf): header = b'HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nContent-Encoding: gzip\r\nContent-Type: text/html\r\nDate: ' header += to_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT')) - header += b'\r\nServer: nginx\r\nTransfer-Encoding: chunked\r\nVary: Accept-Encoding\r\n\r\n' + header += b'\r\nServer: nginx\r\nVary: Accept-Encoding\r\n\r\n' self.has_sent_header = True return header + buf diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 00be536e..5462639f 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -28,6 +28,7 @@ import random import hmac import hashlib +import string from shadowsocks import common from shadowsocks.obfsplugin import plain @@ -40,11 +41,16 @@ def create_tls_obfs(method): def create_tls_auth_obfs(method): return tls_auth(method) +def create_tls_ticket_auth_obfs(method): + return tls_ticket_auth(method) + obfs_map = { 'tls_simple': (create_tls_obfs,), 'tls_simple_compatible': (create_tls_obfs,), 'tls1.0_session_auth': (create_tls_auth_obfs,), 'tls1.0_session_auth_compatible': (create_tls_auth_obfs,), + 'tls1.2_ticket_auth': (create_tls_ticket_auth_obfs,), + 'tls1.2_ticket_auth_compatible': (create_tls_ticket_auth_obfs,), } def match_begin(str1, str2): @@ -268,3 +274,196 @@ def server_decode(self, buf): # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) +class tls_ticket_auth(plain.plain): + def __init__(self, method): + self.method = method + self.handshake_status = 0 + self.send_buffer = b'' + self.recv_buffer = b'' + self.client_id = b'' + self.max_time_dif = 60 * 60 # time dif (second) setting + self.tls_version = b'\x03\x03' + + def init_data(self): + return obfs_auth_data() + + def sni(self, url): + url = common.to_bytes(url) + data = b"\x00" + struct.pack('>H', len(url)) + url + data = b"\x00\x00" + struct.pack('>H', len(data) + 2) + struct.pack('>H', len(data)) + data + return data + + def pack_auth_data(self, client_id): + utc_time = int(time.time()) & 0xFFFFFFFF + data = struct.pack('>I', utc_time) + os.urandom(18) + data += hmac.new(self.server_info.key + client_id, data, hashlib.sha1).digest()[:10] + return data + + def client_encode(self, buf): + if self.handshake_status == -1: + return buf + if self.handshake_status == 8: + return b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf + self.send_buffer += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf + if self.handshake_status == 0: + self.handshake_status = 1 + data = self.tls_version + self.pack_auth_data(self.server_info.data.client_id) + b"\x20" + self.server_info.data.client_id + binascii.unhexlify(b"001cc02bc02fcca9cca8cc14cc13c00ac014c009c013009c0035002f000a" + b"0100") + ext = binascii.unhexlify(b"ff01000100") + host = self.server_info.obfs_param or self.server_info.host + if host and host[-1] in string.digits: + host = '' + ext += self.sni(host) + ext += b"\x00\x17\x00\x00" + ext += b"\x00\x23\x00\xd0" + os.urandom(208) # ticket + ext += binascii.unhexlify(b"000d001600140601060305010503040104030301030302010203") + ext += binascii.unhexlify(b"000500050100000000") + ext += binascii.unhexlify(b"00120000") + ext += binascii.unhexlify(b"75500000") + ext += binascii.unhexlify(b"000b00020100") + ext += binascii.unhexlify(b"000a0006000400170018") + data += struct.pack('>H', len(ext)) + ext + data = b"\x01\x00" + struct.pack('>H', len(data)) + data + data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data + return data + elif self.handshake_status == 1 and len(buf) == 0: + data = b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + "\x00\x20" + os.urandom(22) #Finished + data += hmac.new(self.server_info.key + self.server_info.data.client_id, data, hashlib.sha1).digest()[:10] + ret = data + self.send_buffer + self.send_buffer = b'' + self.handshake_status = 8 + return ret + return b'' + + def client_decode(self, buf): + if self.handshake_status == -1: + return (buf, False) + + if self.handshake_status == 8: + ret = b'' + self.recv_buffer += buf + while len(self.recv_buffer) > 5: + if ord(self.recv_buffer[0]) != 0x17: + logging.error("data = %s" % (binascii.hexlify(self.recv_buffer))) + raise Exception('server_decode appdata error') + size = struct.unpack('>H', self.recv_buffer[3:5])[0] + if len(self.recv_buffer) < size + 5: + break + buf = self.recv_buffer[5:size+5] + ret += buf + self.recv_buffer = self.recv_buffer[size+5:] + return (ret, False) + + if len(buf) < 11 + 32 + 1 + 32: + raise Exception('client_decode data error') + verify = buf[11:33] + if hmac.new(self.server_info.key + self.server_info.data.client_id, verify, hashlib.sha1).digest()[:10] != buf[33:43]: + raise Exception('client_decode data error') + return (b'', True) + + def server_encode(self, buf): + if self.handshake_status == -1: + return buf + if self.handshake_status == 8: + ret = b'' + while len(buf) > 8192: + ret += b"\x17" + self.tls_version + struct.pack('>H', 8192) + buf[:8192] + buf = buf[8192:] + if len(buf) > 0: + ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf + return ret + self.handshake_status = 3 + data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"c02f000005ff01000100") + data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello + data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data + data += b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + "\x00\x20" + os.urandom(22) #Finished + data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] + return data + + def decode_error_return(self, buf): + self.handshake_status = -1 + if self.method == 'tls1.2_session_auth': + return (b'E', False, False) + return (buf, True, False) + + def server_decode(self, buf): + if self.handshake_status == -1: + return (buf, True, False) + + if self.handshake_status == 8: + ret = b'' + self.recv_buffer += buf + while len(self.recv_buffer) > 5: + if ord(self.recv_buffer[0]) != 0x17: + logging.error("data = %s" % (binascii.hexlify(self.recv_buffer))) + raise Exception('server_decode appdata error') + size = struct.unpack('>H', self.recv_buffer[3:5])[0] + if len(self.recv_buffer) < size + 5: + break + ret += self.recv_buffer[5:size+5] + self.recv_buffer = self.recv_buffer[size+5:] + return (ret, True, False) + + if self.handshake_status == 3: + verify = buf + verify_len = 43 - 10 + if len(buf) < 43: + raise Exception('server_decode data error') + if not match_begin(buf, b"\x14" + self.tls_version + "\x00\x01\x01"): #ChangeCipherSpec + raise Exception('server_decode data error') + buf = buf[6:] + if not match_begin(buf, b"\x16" + self.tls_version + "\x00\x20"): #Finished + raise Exception('server_decode data error') + if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: + raise Exception('server_decode data error') + if len(buf) < 37: + raise Exception('server_decode data error') + self.recv_buffer = buf[37:] + self.handshake_status = 8 + return self.server_decode(b'') + + self.handshake_status = 2 + ogn_buf = buf + if not match_begin(buf, b'\x16\x03\x01'): + return self.decode_error_return(ogn_buf) + buf = buf[3:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(ogn_buf) + buf = buf[2:] + if not match_begin(buf, b'\x01\x00'): #client hello + return self.decode_error_return(ogn_buf) + buf = buf[2:] + if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + return self.decode_error_return(ogn_buf) + buf = buf[2:] + if not match_begin(buf, self.tls_version): + return self.decode_error_return(ogn_buf) + buf = buf[2:] + verifyid = buf[:32] + buf = buf[32:] + sessionid_len = ord(buf[0]) + if sessionid_len < 32: + logging.error("tls_auth wrong sessionid_len") + return self.decode_error_return(ogn_buf) + sessionid = buf[1:sessionid_len + 1] + buf = buf[sessionid_len+1:] + self.client_id = sessionid + sha1 = hmac.new(self.server_info.key + sessionid, verifyid[:22], hashlib.sha1).digest()[:10] + utc_time = struct.unpack('>I', verifyid[:4])[0] + time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time) + if time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ + or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: + logging.debug("tls_auth wrong time") + return self.decode_error_return(ogn_buf) + if sha1 != verifyid[22:]: + logging.debug("tls_auth wrong sha1") + return self.decode_error_return(ogn_buf) + if self.server_info.data.client_data.get(verifyid[:22]): + logging.error("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) + return self.decode_error_return(ogn_buf) + self.server_info.data.client_data.sweep() + self.server_info.data.client_data[verifyid[:22]] = sessionid + # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) + return (b'', False, True) + diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c07b28cb..c77faf58 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -282,9 +282,6 @@ def _write_to_sock(self, data, sock): if sock == self._remote_sock: self._server.server_transfer_ul += len(data) self._update_activity(len(data)) - elif not self._is_local and self._obfs is not None: - obfs_encode = self._obfs.server_encode(data) - data = obfs_encode if data: l = len(data) s = sock.send(data) @@ -612,7 +609,8 @@ def _on_local_read(self): self.destroy() return if obfs_decode[2]: - self._write_to_sock(b'', self._local_sock) + data = self._obfs.server_encode(b'') + self._write_to_sock(data, self._local_sock) if obfs_decode[1]: if not self._protocol.obfs.server_info.recv_iv: iv_len = len(self._protocol.obfs.server_info.iv) @@ -704,6 +702,7 @@ def _on_remote_read(self, is_remote_sock): if self._encrypt_correct: data = self._protocol.server_pre_encrypt(data) data = self._encryptor.encrypt(data) + data = self._obfs.server_encode(data) self._update_activity(len(data)) self._server.server_transfer_dl += len(data) else: From d6640b2e81eb93e9fc14b33a07023f2f61534c70 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 18 Apr 2016 22:09:20 +0800 Subject: [PATCH 154/487] add udp data transfer --- server_pool.py | 8 ++++++++ shadowsocks/udprelay.py | 12 ++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/server_pool.py b/server_pool.py index 6c751c16..a3bad6e8 100644 --- a/server_pool.py +++ b/server_pool.py @@ -215,14 +215,22 @@ def get_server_transfer(self, port): if port in self.tcp_servers_pool: ret[0] = self.tcp_servers_pool[port].server_transfer_ul ret[1] = self.tcp_servers_pool[port].server_transfer_dl + if port in self.udp_servers_pool: + ret[0] += self.udp_servers_pool[port].server_transfer_ul + ret[1] += self.udp_servers_pool[port].server_transfer_dl if port in self.tcp_ipv6_servers_pool: ret[0] += self.tcp_ipv6_servers_pool[port].server_transfer_ul ret[1] += self.tcp_ipv6_servers_pool[port].server_transfer_dl + if port in self.udp_ipv6_servers_pool: + ret[0] += self.udp_ipv6_servers_pool[port].server_transfer_ul + ret[1] += self.udp_ipv6_servers_pool[port].server_transfer_dl return ret def get_servers_transfer(self): servers = self.tcp_servers_pool.copy() servers.update(self.tcp_ipv6_servers_pool) + servers.update(self.udp_servers_pool) + servers.update(self.udp_ipv6_servers_pool) ret = {} for port in servers.keys(): ret[port] = self.get_server_transfer(port) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index d5cab1bd..053a0fed 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -529,6 +529,7 @@ def _on_remote_read(self): self.destroy() return try: + self._server.server_transfer_dl += len(data) recv_data = data beg_pos = 0 max_len = len(recv_data) @@ -546,7 +547,6 @@ def _on_remote_read(self): post_data = self._pack_post_data(CMD_POST, pack_id, data) addr = self.get_local_address() self._write_to_sock(post_data, self._local_sock, addr) - self._server.server_transfer_dl += len(post_data) if pack_id <= DOUBLE_SEND_BEG_IDS: post_data = self._pack_post_data(CMD_POST, pack_id, data) self._write_to_sock(post_data, self._local_sock, addr) @@ -1026,6 +1026,7 @@ def _handle_server(self): #(cmd, request_id, data) #logging.info("UDP data %d %d %s" % (data[0], data[1], binascii.hexlify(data[2]))) try: + self.server_transfer_ul += len(data[2]) if data[0] == 0: if len(data[2]) >= 4: for i in range(64): @@ -1152,6 +1153,7 @@ def _handle_server(self): try: #logging.info('UDP handle_server sendto %s:%d %d bytes' % (common.to_str(server_addr), server_port, len(data))) client.sendto(data, (server_addr, server_port)) + self.server_transfer_ul += len(data) except IOError as e: err = eventloop.errno_from_exception(e) if err in (errno.EINPROGRESS, errno.EAGAIN): @@ -1196,6 +1198,7 @@ def _handle_client(self, sock): response = b'\x00\x00\x00' + data client_addr = self._client_fd_to_server_addr.get(sock.fileno()) if client_addr: + self.server_transfer_dl += len(response) self.write_to_server_socket(response, client_addr) else: # this packet is from somewhere else we know @@ -1203,15 +1206,9 @@ def _handle_client(self, sock): pass def write_to_server_socket(self, data, addr): - #self._server_socket.sendto(data, addr) - #''' uncomplete = False retry = 0 try: - #""" - #if self._data_to_write_to_server_socket: - # self._data_to_write_to_server_socket.append([(data, addr), 0]) - #else: self._server_socket.sendto(data, addr) data = None while self._data_to_write_to_server_socket: @@ -1220,7 +1217,6 @@ def write_to_server_socket(self, data, addr): del self._data_to_write_to_server_socket[0] data, addr = data_buf[0] self._server_socket.sendto(data, addr) - #""" except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) uncomplete = True From 9f4f42eefde54cee28a9d2a2ac9ac63595923013 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 22 Apr 2016 23:48:41 +0800 Subject: [PATCH 155/487] catch exit exception --- server_pool.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server_pool.py b/server_pool.py index a3bad6e8..d7fb4b3d 100644 --- a/server_pool.py +++ b/server_pool.py @@ -86,6 +86,10 @@ def _loop(loop, dns_resolver, mgr): import traceback traceback.print_exc() os.exit(0) + except Exception as e: + logging.error(e) + import traceback + traceback.print_exc() def server_is_run(self, port): port = int(port) From b8feaacbbb95e2fa4f4d7fdcd4870cc7ac0c26c5 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 29 Apr 2016 19:20:24 +0800 Subject: [PATCH 156/487] fix mysql update BUG hot update mysql connection info more tls1.2 info for debugging --- Config.py | 1 + db_transfer.py | 5 +++-- shadowsocks/obfsplugin/obfs_tls.py | 26 ++++++++++++++++---------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Config.py b/Config.py index 6323abe4..eb81abfa 100644 --- a/Config.py +++ b/Config.py @@ -5,6 +5,7 @@ MYSQL_PASS = 'ss' MYSQL_DB = 'shadowsocks' MYSQL_TRANSFER_MUL = 1.0 +MYSQL_UPDATE_TIME = 15 MANAGE_PASS = 'ss233333333' #if you want manage in other server you should set this value to global ip diff --git a/db_transfer.py b/db_transfer.py index 6e9ad17e..72f975ce 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -50,7 +50,6 @@ def push_db_all_user(self): dt_transfer[id] = [int(curr_transfer[id][0] * Config.MYSQL_TRANSFER_MUL), int(curr_transfer[id][1] * Config.MYSQL_TRANSFER_MUL)] - self.last_get_transfer = curr_transfer query_head = 'UPDATE user' query_sub_when = '' query_sub_when2 = '' @@ -79,6 +78,7 @@ def push_db_all_user(self): cur.close() conn.commit() conn.close() + self.last_get_transfer = curr_transfer @staticmethod def pull_db_all_user(): @@ -170,6 +170,7 @@ def thread_db(): last_rows = [] try: while True: + reload(Config) try: DbTransfer.get_instance().push_db_all_user() rows = DbTransfer.get_instance().pull_db_all_user() @@ -179,7 +180,7 @@ def thread_db(): trace = traceback.format_exc() logging.error(trace) #logging.warn('db thread except:%s' % e) - if DbTransfer.get_instance().event.wait(15): + if DbTransfer.get_instance().event.wait(Config.MYSQL_UPDATE_TIME): break except KeyboardInterrupt as e: pass diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 5462639f..279e0838 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -251,7 +251,7 @@ def server_decode(self, buf): buf = buf[32:] sessionid_len = ord(buf[0]) if sessionid_len < 32: - logging.error("tls_auth wrong sessionid_len") + logging.info("tls_auth wrong sessionid_len") return self.decode_error_return(ogn_buf) sessionid = buf[1:sessionid_len + 1] buf = buf[sessionid_len+1:] @@ -261,13 +261,13 @@ def server_decode(self, buf): time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time) if time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: - logging.debug("tls_auth wrong time") + logging.info("tls_auth wrong time") return self.decode_error_return(ogn_buf) if sha1 != verifyid[22:]: - logging.debug("tls_auth wrong sha1") + logging.info("tls_auth wrong sha1") return self.decode_error_return(ogn_buf) if self.server_info.data.client_data.get(verifyid[:22]): - logging.error("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) + logging.info("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) return self.decode_error_return(ogn_buf) self.server_info.data.client_data.sweep() self.server_info.data.client_data[verifyid[:22]] = sessionid @@ -344,7 +344,7 @@ def client_decode(self, buf): self.recv_buffer += buf while len(self.recv_buffer) > 5: if ord(self.recv_buffer[0]) != 0x17: - logging.error("data = %s" % (binascii.hexlify(self.recv_buffer))) + logging.info("data = %s" % (binascii.hexlify(self.recv_buffer))) raise Exception('server_decode appdata error') size = struct.unpack('>H', self.recv_buffer[3:5])[0] if len(self.recv_buffer) < size + 5: @@ -396,7 +396,7 @@ def server_decode(self, buf): self.recv_buffer += buf while len(self.recv_buffer) > 5: if ord(self.recv_buffer[0]) != 0x17: - logging.error("data = %s" % (binascii.hexlify(self.recv_buffer))) + logging.info("data = %s" % (binascii.hexlify(self.recv_buffer))) raise Exception('server_decode appdata error') size = struct.unpack('>H', self.recv_buffer[3:5])[0] if len(self.recv_buffer) < size + 5: @@ -423,28 +423,34 @@ def server_decode(self, buf): self.handshake_status = 8 return self.server_decode(b'') + #raise Exception("handshake data = %s" % (binascii.hexlify(buf))) self.handshake_status = 2 ogn_buf = buf if not match_begin(buf, b'\x16\x03\x01'): + logging.info("tls_auth wrong tls head") return self.decode_error_return(ogn_buf) buf = buf[3:] if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + logging.info("tls_auth wrong tls head size") return self.decode_error_return(ogn_buf) buf = buf[2:] if not match_begin(buf, b'\x01\x00'): #client hello + logging.info("tls_auth not client hello message") return self.decode_error_return(ogn_buf) buf = buf[2:] if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: + logging.info("tls_auth wrong message size") return self.decode_error_return(ogn_buf) buf = buf[2:] if not match_begin(buf, self.tls_version): + logging.info("tls_auth wrong tls version") return self.decode_error_return(ogn_buf) buf = buf[2:] verifyid = buf[:32] buf = buf[32:] sessionid_len = ord(buf[0]) if sessionid_len < 32: - logging.error("tls_auth wrong sessionid_len") + logging.info("tls_auth wrong sessionid_len") return self.decode_error_return(ogn_buf) sessionid = buf[1:sessionid_len + 1] buf = buf[sessionid_len+1:] @@ -454,13 +460,13 @@ def server_decode(self, buf): time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time) if time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: - logging.debug("tls_auth wrong time") + logging.info("tls_auth wrong time") return self.decode_error_return(ogn_buf) if sha1 != verifyid[22:]: - logging.debug("tls_auth wrong sha1") + logging.info("tls_auth wrong sha1") return self.decode_error_return(ogn_buf) if self.server_info.data.client_data.get(verifyid[:22]): - logging.error("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) + logging.info("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) return self.decode_error_return(ogn_buf) self.server_info.data.client_data.sweep() self.server_info.data.client_data[verifyid[:22]] = sessionid From 6486f02fbe3389062c26fd843a22fb162c94705f Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 3 May 2016 19:42:32 +0800 Subject: [PATCH 157/487] remove a log info --- shadowsocks/obfsplugin/obfs_tls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 279e0838..00ca0454 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -427,7 +427,6 @@ def server_decode(self, buf): self.handshake_status = 2 ogn_buf = buf if not match_begin(buf, b'\x16\x03\x01'): - logging.info("tls_auth wrong tls head") return self.decode_error_return(ogn_buf) buf = buf[3:] if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: From 0e26f9b513868bad7f1a03512ec776a19bf7dac4 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 7 May 2016 20:08:32 +0800 Subject: [PATCH 158/487] remove useless code --- shadowsocks/obfsplugin/http_simple.py | 91 --------------------------- 1 file changed, 91 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 4fd5f450..3af8f66e 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -34,17 +34,12 @@ def create_http_obfs(method): return http_simple(method) -def create_http2_obfs(method): - return http2_simple(method) - def create_random_head_obfs(method): return random_head(method) obfs_map = { 'http_simple': (create_http_obfs,), 'http_simple_compatible': (create_http_obfs,), - 'http2_simple': (create_http2_obfs,), - 'http2_simple_compatible': (create_http2_obfs,), 'random_head': (create_random_head_obfs,), 'random_head_compatible': (create_random_head_obfs,), } @@ -181,92 +176,6 @@ def server_decode(self, buf): else: return (b'', True, False) -class http2_simple(plain.plain): - def __init__(self, method): - self.method = method - self.has_sent_header = False - self.has_recv_header = False - self.raw_trans_sent = False - self.host = None - self.port = 0 - self.recv_buffer = b'' - self.send_buffer = b'' - - def client_encode(self, buf): - if self.raw_trans_sent: - return buf - self.send_buffer += buf - if not self.has_sent_header: - port = b'' - if self.server_info.port != 80: - port = b':' + common.to_bytes(str(self.server_info.port)) - self.has_sent_header = True - http_head = b"GET / HTTP/1.1\r\n" - http_head += b"Host: " + (self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" - http_head += b"Connection: Upgrade, HTTP2-Settings\r\nUpgrade: h2c\r\n" - http_head += b"HTTP2-Settings: " + base64.urlsafe_b64encode(buf) + b"\r\n" - return http_head + b"\r\n" - if self.has_recv_header: - ret = self.send_buffer - self.send_buffer = b'' - self.raw_trans_sent = True - return ret - return b'' - - def client_decode(self, buf): - if self.has_recv_header: - return (buf, False) - pos = buf.find(b'\r\n\r\n') - if pos >= 0: - self.has_recv_header = True - return (buf[pos + 4:], False) - else: - return (b'', False) - - def server_encode(self, buf): - if self.has_sent_header: - return buf - - header = b'HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: h2c\r\n\r\n' - self.has_sent_header = True - return header + buf - - def not_match_return(self, buf): - self.has_sent_header = True - self.has_recv_header = True - if self.method == 'http2_simple': - return (b'E', False, False) - return (buf, True, False) - - def server_decode(self, buf): - if self.has_recv_header: - return (buf, True, False) - - self.recv_buffer += buf - buf = self.recv_buffer - if len(buf) > 10: - if match_begin(buf, b'GET /'): - pass - else: #not http header, run on original protocol - self.recv_buffer = None - return self.not_match_return(buf) - else: - return (b'', True, False) - - datas = buf.split(b'\r\n\r\n', 1) - if datas and len(datas) > 1 and len(datas[0]) >= 4: - lines = buf.split(b'\r\n') - if lines and len(lines) >= 4: - if match_begin(lines[4], b'HTTP2-Settings: '): - ret_buf = base64.urlsafe_b64decode(lines[4][16:]) - ret_buf += datas[1] - self.has_recv_header = True - return (ret_buf, True, False) - return (b'', True, False) - else: - return (b'', True, False) - return self.not_match_return(buf) - class random_head(plain.plain): def __init__(self, method): self.method = method From 691aa0970a6cffa360608f93a481b0fa9464eb1a Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 16 May 2016 15:54:58 +0800 Subject: [PATCH 159/487] UDP use only 1 lru cache (_client_fd_to_server_addr set to dict type) --- shadowsocks/udprelay.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 053a0fed..287d4c83 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -693,7 +693,7 @@ def handle_client(self, addr, cmd, request_id, data): self._stage = STAGE_DNS self._dns_resolver.resolve(remote_addr, self._handle_dns_resolved) - logging.info('TCP connect %s:%d from %s:%d' % (remote_addr, remote_port, addr[0], addr[1])) + logging.info('TCPonUDP connect %s:%d from %s:%d' % (remote_addr, remote_port, addr[0], addr[1])) else: # ileagal request rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) @@ -882,8 +882,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._is_local = is_local self._cache = lru_cache.LRUCache(timeout=config['timeout'], close_callback=self._close_client) - self._client_fd_to_server_addr = \ - lru_cache.LRUCache(timeout=config['timeout']) + self._client_fd_to_server_addr = {} self._dns_cache = lru_cache.LRUCache(timeout=300) self._eventloop = None self._closed = False @@ -945,6 +944,7 @@ def _close_client(self, client): if hasattr(client, 'close'): self._sockets.remove(client.fileno()) self._eventloop.remove(client) + del self._client_fd_to_server_addr[client.fileno()] client.close() else: # just an address @@ -1330,15 +1330,14 @@ def handle_periodic(self): if self._server_socket: self._server_socket.close() self._server_socket = None - for sock in self._sockets: - sock.close() + #for sock in self._sockets: + # sock.close() logging.info('closed UDP port %d', self._listen_port) before_sweep_size = len(self._sockets) self._cache.sweep() self._dns_cache.sweep() if before_sweep_size != len(self._sockets): logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) - self._client_fd_to_server_addr.sweep() self._sweep_timeout() def close(self, next_tick=False): From bb916eb70535b88ba78fdb7500e84714f1911259 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 16 May 2016 16:56:10 +0800 Subject: [PATCH 160/487] LRUCache add 'clear' method refine close --- shadowsocks/lru_cache.py | 19 +++++++++++++++++++ shadowsocks/tcprelay.py | 2 +- shadowsocks/udprelay.py | 3 +-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 32bab665..2a161dfd 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -107,6 +107,25 @@ def sweep(self): logging.debug('%d keys swept' % c) return c < SWEEP_MAX_ITEMS + def clear(self): + now = time.time() + c = 0 + while c < SWEEP_MAX_ITEMS: + if len(self._keys_to_last_time) == 0: + break + for key in self._keys_to_last_time: + break + last_t = self._keys_to_last_time[key] + value = self._store[key] + if self.close_callback is not None: + self.close_callback(value) + del self._store[key] + del self._keys_to_last_time[key] + c += 1 + if c: + logging.debug('%d keys swept' % c) + return c < SWEEP_MAX_ITEMS + def test(): c = LRUCache(timeout=0.3) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c77faf58..1398e80b 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -497,7 +497,7 @@ def _create_remote_socket(self, ip, port): if len(addrs) == 0: raise Exception("getaddrinfo failed for %s:%d" % (ip, port)) af, socktype, proto, canonname, sa = addrs[0] - if self._forbidden_iplist: + if not self._remote_udp and self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: raise Exception('IP %s is in forbidden list, reject' % common.to_str(sa[0])) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 287d4c83..e4d515a6 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1344,9 +1344,8 @@ def close(self, next_tick=False): logging.debug('UDP close') self._closed = True if not next_tick: + self._cache.clear() if self._eventloop: self._eventloop.remove_periodic(self.handle_periodic) self._eventloop.remove(self._server_socket) self._server_socket.close() - for client in list(self._cache.values()): - client.close() From 4e43a4932d9d5855b4665a2d5842cc8305eab85e Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 16 May 2016 18:20:57 +0800 Subject: [PATCH 161/487] limit active UDP client to 256 --- shadowsocks/lru_cache.py | 4 ++-- shadowsocks/udprelay.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 2a161dfd..9017f81c 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -107,10 +107,10 @@ def sweep(self): logging.debug('%d keys swept' % c) return c < SWEEP_MAX_ITEMS - def clear(self): + def clear(self, keep): now = time.time() c = 0 - while c < SWEEP_MAX_ITEMS: + while len(self._keys_to_last_time) > keep: if len(self._keys_to_last_time) == 0: break for key in self._keys_to_last_time: diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index e4d515a6..a4587b82 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1138,6 +1138,8 @@ def _handle_server(self): (common.to_str(server_addr), server_port, r_addr[0], r_addr[1])) + self._cache.clear(256) + if self._is_local: ref_iv = [encrypt.encrypt_new_iv(self._method)] self._protocol.obfs.server_info.iv = ref_iv[0] @@ -1344,7 +1346,7 @@ def close(self, next_tick=False): logging.debug('UDP close') self._closed = True if not next_tick: - self._cache.clear() + self._cache.clear(0) if self._eventloop: self._eventloop.remove_periodic(self.handle_periodic) self._eventloop.remove(self._server_socket) From d47b0d47ea386808e40a5bc454afc029f94ac0d7 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 May 2016 17:27:33 +0800 Subject: [PATCH 162/487] add config "udp_timeout" drop TCP over UDP query --- shadowsocks/shell.py | 1 + shadowsocks/udprelay.py | 134 +++++++++++++++++++++------------------- 2 files changed, 70 insertions(+), 65 deletions(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 342b8da5..775b39c6 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -236,6 +236,7 @@ def get_config(is_local): config['obfs_param'] = to_str(config.get('obfs_param', '')) config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) + config['udp_timeout'] = int(config.get('udp_timeout', config['timeout'])) config['fast_open'] = config.get('fast_open', False) config['workers'] = config.get('workers', 1) config['pid-file'] = config.get('pid-file', '/var/run/shadowsocks.pid') diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index a4587b82..713c3465 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -880,7 +880,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._method = config['method'] self._timeout = config['timeout'] self._is_local = is_local - self._cache = lru_cache.LRUCache(timeout=config['timeout'], + self._cache = lru_cache.LRUCache(timeout=config['udp_timeout'], close_callback=self._close_client) self._client_fd_to_server_addr = {} self._dns_cache = lru_cache.LRUCache(timeout=300) @@ -1023,69 +1023,8 @@ def _handle_server(self): return if type(data) is tuple: - #(cmd, request_id, data) - #logging.info("UDP data %d %d %s" % (data[0], data[1], binascii.hexlify(data[2]))) - try: - self.server_transfer_ul += len(data[2]) - if data[0] == 0: - if len(data[2]) >= 4: - for i in range(64): - req_id = random.randint(1, 65535) - if req_id not in self._reqid_to_hd: - break - if req_id in self._reqid_to_hd: - for i in range(64): - req_id = random.randint(1, 65535) - if type(self._reqid_to_hd[req_id]) is tuple: - break - # return req id - self._reqid_to_hd[req_id] = (data[2][0:4], None) - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, req_id, RSP_STATE_CONNECTED) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - elif data[0] == CMD_CONNECT_REMOTE: - if len(data[2]) > 4 and data[1] in self._reqid_to_hd: - # create - if type(self._reqid_to_hd[data[1]]) is tuple: - if data[2][0:4] == self._reqid_to_hd[data[1]][0]: - handle = TCPRelayHandler(self, self._reqid_to_hd, self._fd_to_handlers, - self._eventloop, self._server_socket, - self._reqid_to_hd[data[1]][0], self._reqid_to_hd[data[1]][1], - self._config, self._dns_resolver, self._is_local) - self._reqid_to_hd[data[1]] = handle - handle.handle_client(r_addr, CMD_CONNECT, data[1], data[2]) - handle.handle_client(r_addr, *data) - self.update_activity(handle) - else: - # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - else: - self.update_activity(self._reqid_to_hd[data[1]]) - self._reqid_to_hd[data[1]].handle_client(r_addr, *data) - else: - # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - elif data[0] > CMD_CONNECT_REMOTE and data[0] <= CMD_DISCONNECT: - if data[1] in self._reqid_to_hd: - if type(self._reqid_to_hd[data[1]]) is tuple: - pass - else: - self.update_activity(self._reqid_to_hd[data[1]]) - self._reqid_to_hd[data[1]].handle_client(r_addr, *data) - else: - # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - return - except Exception as e: - trace = traceback.format_exc() - logging.error(trace) - return + return + return self._handle_tcp_over_udp(data, r_addr) try: header_result = parse_header(data) @@ -1105,6 +1044,7 @@ def _handle_server(self): addrs = self._dns_cache.get(server_addr, None) if addrs is None: + # TODO async getaddrinfo addrs = socket.getaddrinfo(server_addr, server_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) if not addrs: @@ -1117,7 +1057,6 @@ def _handle_server(self): key = client_key(r_addr, af) client = self._cache.get(key, None) if not client: - # TODO async getaddrinfo if self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: logging.debug('IP %s is in forbidden list, drop' % @@ -1163,6 +1102,71 @@ def _handle_server(self): else: shell.print_exception(e) + def _handle_tcp_over_udp(self, data, r_addr): + #(cmd, request_id, data) + #logging.info("UDP data %d %d %s" % (data[0], data[1], binascii.hexlify(data[2]))) + try: + self.server_transfer_ul += len(data[2]) + if data[0] == 0: + if len(data[2]) >= 4: + for i in range(64): + req_id = random.randint(1, 65535) + if req_id not in self._reqid_to_hd: + break + if req_id in self._reqid_to_hd: + for i in range(64): + req_id = random.randint(1, 65535) + if type(self._reqid_to_hd[req_id]) is tuple: + break + # return req id + self._reqid_to_hd[req_id] = (data[2][0:4], None) + rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, req_id, RSP_STATE_CONNECTED) + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + elif data[0] == CMD_CONNECT_REMOTE: + if len(data[2]) > 4 and data[1] in self._reqid_to_hd: + # create + if type(self._reqid_to_hd[data[1]]) is tuple: + if data[2][0:4] == self._reqid_to_hd[data[1]][0]: + handle = TCPRelayHandler(self, self._reqid_to_hd, self._fd_to_handlers, + self._eventloop, self._server_socket, + self._reqid_to_hd[data[1]][0], self._reqid_to_hd[data[1]][1], + self._config, self._dns_resolver, self._is_local) + self._reqid_to_hd[data[1]] = handle + handle.handle_client(r_addr, CMD_CONNECT, data[1], data[2]) + handle.handle_client(r_addr, *data) + self.update_activity(handle) + else: + # disconnect + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + else: + self.update_activity(self._reqid_to_hd[data[1]]) + self._reqid_to_hd[data[1]].handle_client(r_addr, *data) + else: + # disconnect + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + elif data[0] > CMD_CONNECT_REMOTE and data[0] <= CMD_DISCONNECT: + if data[1] in self._reqid_to_hd: + if type(self._reqid_to_hd[data[1]]) is tuple: + pass + else: + self.update_activity(self._reqid_to_hd[data[1]]) + self._reqid_to_hd[data[1]].handle_client(r_addr, *data) + else: + # disconnect + rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) + data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) + self.write_to_server_socket(data_to_send, r_addr) + return + except Exception as e: + trace = traceback.format_exc() + logging.error(trace) + return + def _handle_client(self, sock): data, r_addr = sock.recvfrom(BUF_SIZE) if not data: From b8cf491212186d9e1c0a7c235b742d5ba96c3561 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 May 2016 18:00:17 +0800 Subject: [PATCH 163/487] remove mgr --- server_pool.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server_pool.py b/server_pool.py index d7fb4b3d..40304778 100644 --- a/server_pool.py +++ b/server_pool.py @@ -55,7 +55,7 @@ def __init__(self): if not self.config.get('dns_ipv6', False): asyncdns.IPV6_CONNECTION_SUPPORT = False - self.mgr = asyncmgr.ServerMgr() + self.mgr = None #asyncmgr.ServerMgr() self.tcp_servers_pool = {} self.tcp_ipv6_servers_pool = {} @@ -78,7 +78,8 @@ def stop(self): @staticmethod def _loop(loop, dns_resolver, mgr): try: - mgr.add_to_loop(loop) + if mgr is not None: + mgr.add_to_loop(loop) dns_resolver.add_to_loop(loop) loop.run() except (KeyboardInterrupt, IOError, OSError) as e: @@ -211,7 +212,7 @@ def cb_del_server(self, port): except Exception as e: logging.warn(e) - return True + return True def get_server_transfer(self, port): port = int(port) From b71af8f4df4ccde55d95f5a1eaed551a37030a50 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 May 2016 23:12:37 +0800 Subject: [PATCH 164/487] log connection info as debug level by default --- shadowsocks/common.py | 1 + shadowsocks/shell.py | 1 + shadowsocks/tcprelay.py | 9 ++++++--- shadowsocks/udprelay.py | 19 ++++++++++++++----- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 39903fd5..68017d52 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -40,6 +40,7 @@ def compat_chr(d): ord = compat_ord chr = compat_chr +connect_log = logging.debug def to_bytes(s): if bytes != str: diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 775b39c6..c4b73813 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -242,6 +242,7 @@ def get_config(is_local): config['pid-file'] = config.get('pid-file', '/var/run/shadowsocks.pid') config['log-file'] = config.get('log-file', '/var/log/shadowsocks.log') config['verbose'] = config.get('verbose', False) + config['connect_verbose_info'] = config.get('connect_verbose_info', 0) config['local_address'] = to_str(config.get('local_address', '127.0.0.1')) config['local_port'] = config.get('local_port', 1080) if is_local: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 1398e80b..1680a407 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -450,7 +450,7 @@ def _handle_stage_addr(self, ogn_data, data): data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) connecttype, remote_addr, remote_port, header_length = header_result - logging.info('%s connecting %s:%d from %s:%d' % + common.connect_log('%s connecting %s:%d from %s:%d' % ((connecttype == 0) and 'TCP' or 'UDP', common.to_str(remote_addr), remote_port, self._client_address[0], self._client_address[1])) @@ -849,6 +849,9 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self.protocol_data = obfs.obfs(config['protocol']).init_data() self.obfs_data = obfs.obfs(config['obfs']).init_data() + if config.get('connect_verbose_info', 0) > 0: + common.connect_log = logging.info + self._timeout = config['timeout'] self._timeouts = [] # a list for all the handlers # we trim the timeouts once a while @@ -938,10 +941,10 @@ def _sweep_timeout(self): break else: if handler.remote_address: - logging.warn('timed out: %s:%d' % + logging.debug('timed out: %s:%d' % handler.remote_address) else: - logging.warn('timed out') + logging.debug('timed out') handler.destroy() self._timeouts[pos] = None # free memory pos += 1 diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 713c3465..47371993 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -693,7 +693,7 @@ def handle_client(self, addr, cmd, request_id, data): self._stage = STAGE_DNS self._dns_resolver.resolve(remote_addr, self._handle_dns_resolved) - logging.info('TCPonUDP connect %s:%d from %s:%d' % (remote_addr, remote_port, addr[0], addr[1])) + common.connect_log('TCPonUDP connect %s:%d from %s:%d' % (remote_addr, remote_port, addr[0], addr[1])) else: # ileagal request rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) @@ -865,6 +865,8 @@ def client_key(source_addr, server_af): class UDPRelay(object): def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._config = config + if config.get('connect_verbose_info', 0) > 0: + common.connect_log = logging.info if is_local: self._listen_addr = config['local_address'] self._listen_port = config['local_port'] @@ -942,12 +944,19 @@ def _get_a_server(self): def _close_client(self, client): if hasattr(client, 'close'): + if not self._is_local: + if client.fileno() in self._client_fd_to_server_addr: + logging.debug('close_client: %s' % + (self._client_fd_to_server_addr[client.fileno()],)) + else: + client.info('close_client') self._sockets.remove(client.fileno()) self._eventloop.remove(client) del self._client_fd_to_server_addr[client.fileno()] client.close() else: # just an address + client.info('close_client pass %s' % client) pass def _pre_parse_udp_header(self, data): @@ -1073,7 +1082,7 @@ def _handle_server(self): logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) - logging.info('UDP data to %s:%d from %s:%d' % + common.connect_log('UDP data to %s:%d from %s:%d' % (common.to_str(server_addr), server_port, r_addr[0], r_addr[1])) @@ -1083,7 +1092,7 @@ def _handle_server(self): ref_iv = [encrypt.encrypt_new_iv(self._method)] self._protocol.obfs.server_info.iv = ref_iv[0] data = self._protocol.client_udp_pre_encrypt(data) - logging.info("%s" % (binascii.hexlify(data),)) + #logging.debug("%s" % (binascii.hexlify(data),)) data = encrypt.encrypt_all_iv(self._protocol.obfs.server_info.key, self._method, 1, data, ref_iv) if not data: return @@ -1285,10 +1294,10 @@ def _sweep_timeout(self): break else: if handler.remote_address: - logging.warn('timed out: %s:%d' % + logging.debug('timed out: %s:%d' % handler.remote_address) else: - logging.warn('timed out') + logging.debug('timed out') handler.destroy() handler.destroy_local() self._timeouts[pos] = None # free memory From 4a02d1959724460b0bd3ce3ae746473989ed0eef Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 17 May 2016 23:30:00 +0800 Subject: [PATCH 165/487] add udp_cache --- shadowsocks/shell.py | 1 + shadowsocks/udprelay.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index c4b73813..d53e4611 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -237,6 +237,7 @@ def get_config(is_local): config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) config['udp_timeout'] = int(config.get('udp_timeout', config['timeout'])) + config['udp_cache'] = int(config.get('udp_cache', 64)) config['fast_open'] = config.get('fast_open', False) config['workers'] = config.get('workers', 1) config['pid-file'] = config.get('pid-file', '/var/run/shadowsocks.pid') diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 47371993..7fb67b18 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -882,6 +882,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._method = config['method'] self._timeout = config['timeout'] self._is_local = is_local + self._udp_cache_size = config['udp_cache'] self._cache = lru_cache.LRUCache(timeout=config['udp_timeout'], close_callback=self._close_client) self._client_fd_to_server_addr = {} @@ -1086,7 +1087,7 @@ def _handle_server(self): (common.to_str(server_addr), server_port, r_addr[0], r_addr[1])) - self._cache.clear(256) + self._cache.clear(self._udp_cache_size) if self._is_local: ref_iv = [encrypt.encrypt_new_iv(self._method)] From 7e8472332a888bde3aef2dde2a59f562f0d1f9ab Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 18 May 2016 17:05:45 +0800 Subject: [PATCH 166/487] xrange -> range --- db_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index 72f975ce..e50e4596 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -97,7 +97,7 @@ def pull_db_all_user(): rows = [] for r in cur.fetchall(): d = {} - for column in xrange(len(keys)): + for column in range(len(keys)): d[keys[column]] = r[column] rows.append(d) cur.close() From f91e04b0bdf25a6601b849c1ec2752db191c8007 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 18 May 2016 21:55:11 +0800 Subject: [PATCH 167/487] TCP: bind the IP which connecting in, or set the IP by config file allow setting method/bind/bindv6 for each port --- shadowsocks/server.py | 12 ++++++++++++ shadowsocks/tcprelay.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index ddc206c4..40b040a0 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -65,10 +65,13 @@ def main(): config_password = config.get('password', 'm') del config['port_password'] for port, password_obfs in port_password.items(): + method = config["method"] protocol = config.get("protocol", 'origin') protocol_param = config.get("protocol_param", '') obfs = config.get("obfs", 'plain') obfs_param = config.get("obfs_param", '') + bind = config.get("bind", '') + bindv6 = config.get("bindv6", '') if type(password_obfs) == list: password = password_obfs[0] obfs = password_obfs[1] @@ -76,10 +79,13 @@ def main(): protocol = password_obfs[2] elif type(password_obfs) == dict: password = password_obfs.get('password', config_password) + method = password_obfs.get('method', method) protocol = password_obfs.get('protocol', protocol) protocol_param = password_obfs.get('protocol_param', protocol_param) obfs = password_obfs.get('obfs', obfs) obfs_param = password_obfs.get('obfs_param', obfs_param) + bind = password_obfs.get('bind', bind) + bindv6 = password_obfs.get('bindv6', bindv6) else: password = password_obfs a_config = config.copy() @@ -92,10 +98,13 @@ def main(): a_config['server_ipv6'] = a_config['server_ipv6'][1:-1] a_config['server_port'] = int(port) a_config['password'] = password + a_config['method'] = method a_config['protocol'] = protocol a_config['protocol_param'] = protocol_param a_config['obfs'] = obfs a_config['obfs_param'] = obfs_param + a_config['bind'] = bind + a_config['bindv6'] = bindv6 a_config['server'] = a_config['server_ipv6'] logging.info("starting server at [%s]:%d" % (a_config['server'], int(port))) @@ -110,10 +119,13 @@ def main(): a_config = config.copy() a_config['server_port'] = int(port) a_config['password'] = password + a_config['method'] = method a_config['protocol'] = protocol a_config['protocol_param'] = protocol_param a_config['obfs'] = obfs a_config['obfs_param'] = obfs_param + a_config['bind'] = bind + a_config['bindv6'] = bindv6 logging.info("starting server at %s:%d" % (a_config['server'], int(port))) tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 1680a407..254038ca 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -138,6 +138,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._protocol.set_server_info(server_info) self._redir_list = config.get('redirect', ["0.0.0.0:0"]) + self._bind = config.get('bind', '') + self._bindv6 = config.get('bindv6', '') self._fastopen_connected = False self._data_to_write_to_local = [] @@ -516,6 +518,20 @@ def _create_remote_socket(self, ip, port): remote_sock_v6.setblocking(False) else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + if not self._is_local: + bind_addr = '' + if self._bind and af == socket.AF_INET: + bind_addr = self._bind + elif self._bindv6 and af == socket.AF_INET: + bind_addr = self._bindv6 + else: + bind_addr = self._local_sock.getsockname()[0] + + bind_addr = bind_addr.replace("::ffff:", "") + local_addrs = socket.getaddrinfo(bind_addr, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) + if local_addrs[0][0] == af: + logging.debug("bind %s" % (bind_addr,)) + remote_sock.bind((bind_addr, 0)) return remote_sock def _handle_dns_resolved(self, result, error): From 6906c7db04d3f0e6c5fcf948023a5070baa36529 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 18 May 2016 22:10:30 +0800 Subject: [PATCH 168/487] rename bind -> out_bind --- shadowsocks/server.py | 12 ++++++------ shadowsocks/tcprelay.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 40b040a0..3547309e 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -70,8 +70,8 @@ def main(): protocol_param = config.get("protocol_param", '') obfs = config.get("obfs", 'plain') obfs_param = config.get("obfs_param", '') - bind = config.get("bind", '') - bindv6 = config.get("bindv6", '') + bind = config.get("out_bind", '') + bindv6 = config.get("out_bindv6", '') if type(password_obfs) == list: password = password_obfs[0] obfs = password_obfs[1] @@ -103,8 +103,8 @@ def main(): a_config['protocol_param'] = protocol_param a_config['obfs'] = obfs a_config['obfs_param'] = obfs_param - a_config['bind'] = bind - a_config['bindv6'] = bindv6 + a_config['out_bind'] = bind + a_config['out_bindv6'] = bindv6 a_config['server'] = a_config['server_ipv6'] logging.info("starting server at [%s]:%d" % (a_config['server'], int(port))) @@ -124,8 +124,8 @@ def main(): a_config['protocol_param'] = protocol_param a_config['obfs'] = obfs a_config['obfs_param'] = obfs_param - a_config['bind'] = bind - a_config['bindv6'] = bindv6 + a_config['out_bind'] = bind + a_config['out_bindv6'] = bindv6 logging.info("starting server at %s:%d" % (a_config['server'], int(port))) tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 254038ca..7fa4f35c 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -138,8 +138,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._protocol.set_server_info(server_info) self._redir_list = config.get('redirect', ["0.0.0.0:0"]) - self._bind = config.get('bind', '') - self._bindv6 = config.get('bindv6', '') + self._bind = config.get('out_bind', '') + self._bindv6 = config.get('out_bindv6', '') self._fastopen_connected = False self._data_to_write_to_local = [] @@ -522,7 +522,7 @@ def _create_remote_socket(self, ip, port): bind_addr = '' if self._bind and af == socket.AF_INET: bind_addr = self._bind - elif self._bindv6 and af == socket.AF_INET: + elif self._bindv6 and af == socket.AF_INET6: bind_addr = self._bindv6 else: bind_addr = self._local_sock.getsockname()[0] From be922ba7c2e5ebe6f073033ea5cf05fda6a3427c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 18 May 2016 23:41:30 +0800 Subject: [PATCH 169/487] add ignore_bind list --- shadowsocks/shell.py | 6 ++++++ shadowsocks/tcprelay.py | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index d53e4611..69853d08 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -261,6 +261,12 @@ def get_config(is_local): except Exception as e: logging.error(e) sys.exit(2) + try: + config['ignore_bind'] = \ + IPNetwork(config.get('ignore_bind', '127.0.0.0/8,::1/128')) + except Exception as e: + logging.error(e) + sys.exit(2) config['server_port'] = config.get('server_port', 8388) logging.getLogger('').handlers = [] diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7fa4f35c..1a080bb3 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -140,6 +140,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._redir_list = config.get('redirect', ["0.0.0.0:0"]) self._bind = config.get('out_bind', '') self._bindv6 = config.get('out_bindv6', '') + self._ignore_bind_list = config.get('ignore_bind', []) self._fastopen_connected = False self._data_to_write_to_local = [] @@ -528,10 +529,13 @@ def _create_remote_socket(self, ip, port): bind_addr = self._local_sock.getsockname()[0] bind_addr = bind_addr.replace("::ffff:", "") - local_addrs = socket.getaddrinfo(bind_addr, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) - if local_addrs[0][0] == af: - logging.debug("bind %s" % (bind_addr,)) - remote_sock.bind((bind_addr, 0)) + if bind_addr in self._ignore_bind_list: + bind_addr = None + if bind_addr: + local_addrs = socket.getaddrinfo(bind_addr, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) + if local_addrs[0][0] == af: + logging.debug("bind %s" % (bind_addr,)) + remote_sock.bind((bind_addr, 0)) return remote_sock def _handle_dns_resolved(self, result, error): From cb866dc8fe524d15264c2359f743fcb1e8d4d2f2 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 19 May 2016 13:42:52 +0800 Subject: [PATCH 170/487] set udp_timeout default value --- shadowsocks/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 69853d08..babac982 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -236,7 +236,7 @@ def get_config(is_local): config['obfs_param'] = to_str(config.get('obfs_param', '')) config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) - config['udp_timeout'] = int(config.get('udp_timeout', config['timeout'])) + config['udp_timeout'] = int(config.get('udp_timeout', 120)) config['udp_cache'] = int(config.get('udp_cache', 64)) config['fast_open'] = config.get('fast_open', False) config['workers'] = config.get('workers', 1) From 760a5308ddcee9a7e8108573e773b98b52c47b41 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 19 May 2016 14:14:48 +0800 Subject: [PATCH 171/487] add "udp_timeout" in config.json --- config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config.json b/config.json index eaaf50ca..6b521783 100644 --- a/config.json +++ b/config.json @@ -6,6 +6,7 @@ "local_port": 1080, "password": "m", "timeout": 120, + "udp_timeout": 60, "method": "aes-256-cfb", "protocol": "auth_sha1_compatible", "protocol_param": "", From 4e10ff90634adff5293aea823288ab1a0f1487b1 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 20 May 2016 15:49:19 +0800 Subject: [PATCH 172/487] better way to stop a server --- Config.py | 2 +- db_transfer.py | 12 +++++++++++- server_pool.py | 8 ++++---- shadowsocks/tcprelay.py | 5 ++--- shadowsocks/udprelay.py | 22 +++++++++++++--------- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/Config.py b/Config.py index eb81abfa..752ebde8 100644 --- a/Config.py +++ b/Config.py @@ -5,7 +5,7 @@ MYSQL_PASS = 'ss' MYSQL_DB = 'shadowsocks' MYSQL_TRANSFER_MUL = 1.0 -MYSQL_UPDATE_TIME = 15 +MYSQL_UPDATE_TIME = 60 MANAGE_PASS = 'ss233333333' #if you want manage in other server you should set this value to global ip diff --git a/db_transfer.py b/db_transfer.py index e50e4596..d5c6994f 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -115,6 +115,7 @@ def del_server_out_of_bound_safe(last_rows, rows): except Exception as e: logging.error('load switchrule.py fail') cur_servers = {} + new_servers = {} for row in rows: try: allow = switchrule.isTurnOn(row) and row['enable'] == 1 and row['u'] + row['d'] < row['transfer_enable'] @@ -139,9 +140,10 @@ def del_server_out_of_bound_safe(last_rows, rows): #password changed logging.info('db stop server at port [%s] reason: password changed' % (port,)) ServerPool.get_instance().cb_del_server(port) - ServerPool.get_instance().new_server(port, passwd) + new_servers[port] = passwd elif allow and ServerPool.get_instance().server_run_status(port) is False: + #new_servers[port] = passwd logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) ServerPool.get_instance().new_server(port, passwd) @@ -152,6 +154,14 @@ def del_server_out_of_bound_safe(last_rows, rows): logging.info('db stop server at port [%s] reason: port not exist' % (row['port'])) ServerPool.get_instance().cb_del_server(row['port']) + if len(new_servers) > 0: + from shadowsocks import eventloop + DbTransfer.get_instance().event.wait(eventloop.TIMEOUT_PRECISION) + for port in new_servers.keys(): + passwd = new_servers[port] + logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) + ServerPool.get_instance().new_server(port, passwd) + @staticmethod def del_servers(): for port in ServerPool.get_instance().tcp_servers_pool.keys(): diff --git a/server_pool.py b/server_pool.py index 40304778..27e1168e 100644 --- a/server_pool.py +++ b/server_pool.py @@ -186,12 +186,12 @@ def cb_del_server(self, port): else: logging.info("stopped server at %s:%d" % (self.config['server'], port)) try: - self.tcp_servers_pool[port].close(False) + self.tcp_servers_pool[port].close(True) del self.tcp_servers_pool[port] except Exception as e: logging.warn(e) try: - self.udp_servers_pool[port].close(False) + self.udp_servers_pool[port].close(True) del self.udp_servers_pool[port] except Exception as e: logging.warn(e) @@ -202,12 +202,12 @@ def cb_del_server(self, port): else: logging.info("stopped server at [%s]:%d" % (self.config['server_ipv6'], port)) try: - self.tcp_ipv6_servers_pool[port].close(False) + self.tcp_ipv6_servers_pool[port].close(True) del self.tcp_ipv6_servers_pool[port] except Exception as e: logging.warn(e) try: - self.udp_ipv6_servers_pool[port].close(False) + self.udp_ipv6_servers_pool[port].close(True) del self.udp_ipv6_servers_pool[port] except Exception as e: logging.warn(e) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 1a080bb3..04336697 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1018,9 +1018,8 @@ def handle_periodic(self): self._server_socket.close() self._server_socket = None logging.info('closed TCP port %d', self._listen_port) - if not self._fd_to_handlers: - logging.info('stopping') - self._eventloop.stop() + for handler in list(self._fd_to_handlers.values()): + handler.destroy() self._sweep_timeout() def close(self, next_tick=False): diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 7fb67b18..4c66ac8d 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1343,25 +1343,29 @@ def handle_event(self, sock, fd, event): def handle_periodic(self): if self._closed: + self._cache.clear(0) + self._dns_cache.sweep() + if self._eventloop: + self._eventloop.remove_periodic(self.handle_periodic) + self._eventloop.remove(self._server_socket) if self._server_socket: self._server_socket.close() self._server_socket = None - #for sock in self._sockets: - # sock.close() logging.info('closed UDP port %d', self._listen_port) - before_sweep_size = len(self._sockets) - self._cache.sweep() - self._dns_cache.sweep() - if before_sweep_size != len(self._sockets): - logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) - self._sweep_timeout() + else: + before_sweep_size = len(self._sockets) + self._cache.sweep() + self._dns_cache.sweep() + if before_sweep_size != len(self._sockets): + logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) + self._sweep_timeout() def close(self, next_tick=False): logging.debug('UDP close') self._closed = True if not next_tick: - self._cache.clear(0) if self._eventloop: self._eventloop.remove_periodic(self.handle_periodic) self._eventloop.remove(self._server_socket) self._server_socket.close() + self._cache.clear(0) From 5dbeac99def8856d66887bb8d559a046852800d0 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 21 May 2016 19:35:45 +0800 Subject: [PATCH 173/487] more error info about peers --- shadowsocks/tcprelay.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 04336697..0fedd80d 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -276,6 +276,7 @@ def _write_to_sock(self, data, sock): uncomplete = True else: shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() return False return True @@ -301,10 +302,12 @@ def _write_to_sock(self, data, sock): else: #traceback.print_exc() shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() return False except Exception as e: shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() return False if uncomplete: @@ -315,14 +318,14 @@ def _write_to_sock(self, data, sock): self._data_to_write_to_remote.append(data) self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) else: - logging.error('write_all_to_sock:unknown socket') + logging.error('write_all_to_sock:unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) else: if sock == self._local_sock: self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) elif sock == self._remote_sock: self._update_stream(STREAM_UP, WAIT_STATUS_READING) else: - logging.error('write_all_to_sock:unknown socket') + logging.error('write_all_to_sock:unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) return True def _get_redirect_host(self, client_address, ogn_data): @@ -400,6 +403,7 @@ def _handle_stage_connecting(self, data): shell.print_exception(e) if self._config['verbose']: traceback.print_exc() + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() def _get_head_size(self, buf, def_value): @@ -600,6 +604,7 @@ def _handle_dns_resolved(self, result, error): shell.print_exception(e) if self._config['verbose']: traceback.print_exc() + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() def _on_local_read(self): @@ -626,6 +631,7 @@ def _on_local_read(self): obfs_decode = self._obfs.server_decode(data) except Exception as e: shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() return if obfs_decode[2]: @@ -642,6 +648,7 @@ def _on_local_read(self): data = self._protocol.server_post_decrypt(data) except Exception as e: shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() else: return @@ -703,6 +710,7 @@ def _on_remote_read(self, is_remote_sock): obfs_decode = self._obfs.client_decode(data) except Exception as e: shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() return if obfs_decode[1]: @@ -716,6 +724,7 @@ def _on_remote_read(self, is_remote_sock): data = self._protocol.client_post_decrypt(data) except Exception as e: shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() return else: @@ -733,7 +742,7 @@ def _on_remote_read(self, is_remote_sock): shell.print_exception(e) if self._config['verbose']: traceback.print_exc() - # TODO use logging when debug completed + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() def _on_local_write(self): @@ -796,7 +805,7 @@ def handle_event(self, sock, event): if event & eventloop.POLL_OUT: self._on_local_write() else: - logging.warn('unknown socket') + logging.warn('unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) def _log_error(self, e): logging.error('%s when handling connection from %s:%d' % From 979a51318ce5df9680fce7d17da45d81614009e2 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 22 May 2016 23:41:03 +0800 Subject: [PATCH 174/487] add http_post --- config.json | 3 +- shadowsocks/obfsplugin/http_simple.py | 80 ++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/config.json b/config.json index 6b521783..30bc51da 100644 --- a/config.json +++ b/config.json @@ -12,8 +12,9 @@ "protocol_param": "", "obfs": "http_simple_compatible", "obfs_param": "", - "redirect": "", "dns_ipv6": false, + "connect_verbose_info": 0, + "redirect": "", "fast_open": false, "workers": 1 } diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 3af8f66e..10cde0f7 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -31,15 +31,20 @@ from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord, chr -def create_http_obfs(method): +def create_http_simple_obfs(method): return http_simple(method) +def create_http_post_obfs(method): + return http_post(method) + def create_random_head_obfs(method): return random_head(method) obfs_map = { - 'http_simple': (create_http_obfs,), - 'http_simple_compatible': (create_http_obfs,), + 'http_simple': (create_http_simple_obfs,), + 'http_simple_compatible': (create_http_simple_obfs,), + 'http_post': (create_http_post_obfs,), + 'http_post_compatible': (create_http_post_obfs,), 'random_head': (create_random_head_obfs,), 'random_head_compatible': (create_random_head_obfs,), } @@ -176,6 +181,75 @@ def server_decode(self, buf): else: return (b'', True, False) +class http_post(http_simple): + def __init__(self, method): + super(http_post, self).__init__(method) + + def boundary(self): + return b''.join([random.choice(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") for i in range(32)]) + + def client_encode(self, buf): + if self.has_sent_header: + return buf + head_size = len(self.server_info.iv) + self.server_info.head_len + if len(buf) - head_size > 64: + headlen = head_size + random.randint(0, 64) + else: + headlen = len(buf) + headdata = buf[:headlen] + buf = buf[headlen:] + port = b'' + if self.server_info.port != 80: + port = b':' + to_bytes(str(self.server_info.port)) + http_head = b"POST /" + self.encode_head(headdata) + b" HTTP/1.1\r\n" + http_head += b"Host: " + to_bytes(self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" + http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" + http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\n" + http_head += b"Content-Type: multipart/form-data; boundary=" + self.boundary() + b"\r\nDNT: 1\r\n" + http_head += "Connection: keep-alive\r\n\r\n" + self.has_sent_header = True + return http_head + buf + + def not_match_return(self, buf): + self.has_sent_header = True + self.has_recv_header = True + if self.method == 'http_post': + return (b'E', False, False) + return (buf, True, False) + + def server_decode(self, buf): + if self.has_recv_header: + return (buf, True, False) + + self.recv_buffer += buf + buf = self.recv_buffer + if len(buf) > 10: + if match_begin(buf, b'GET ') or match_begin(buf, b'POST '): + if len(buf) > 65536: + self.recv_buffer = None + logging.warn('http_post: over size') + return self.not_match_return(buf) + else: #not http header, run on original protocol + self.recv_buffer = None + logging.debug('http_post: not match begin') + return self.not_match_return(buf) + else: + return (b'', True, False) + + if b'\r\n\r\n' in buf: + datas = buf.split(b'\r\n\r\n', 1) + ret_buf = self.get_data_from_http_header(buf) + if len(datas) > 1: + ret_buf += datas[1] + if len(ret_buf) >= 15: + self.has_recv_header = True + return (ret_buf, True, False) + if len(ret_buf) == 0: + return self.not_match_return(buf) + return (b'', True, False) + else: + return (b'', True, False) + class random_head(plain.plain): def __init__(self, method): self.method = method From c5ca6bd80aa2fc7278f7486e06bc050df54d9893 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 23 May 2016 17:11:32 +0800 Subject: [PATCH 175/487] optimize DNS query --- shadowsocks/tcprelay.py | 3 +++ shadowsocks/udprelay.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 0fedd80d..4a456a33 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1008,6 +1008,9 @@ def handle_event(self, sock, fd, event): if error_no in (errno.EAGAIN, errno.EINPROGRESS, errno.EWOULDBLOCK): return + elif error_no == errno.ECONNRESET: + shell.print_exception(e) + logging.info("recv RST, ignore") else: shell.print_exception(e) if self._config['verbose']: diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 4c66ac8d..1b119ea9 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -885,6 +885,8 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): self._udp_cache_size = config['udp_cache'] self._cache = lru_cache.LRUCache(timeout=config['udp_timeout'], close_callback=self._close_client) + self._cache_dns_client = lru_cache.LRUCache(timeout=10, + close_callback=self._close_client) self._client_fd_to_server_addr = {} self._dns_cache = lru_cache.LRUCache(timeout=300) self._eventloop = None @@ -1066,6 +1068,8 @@ def _handle_server(self): af, socktype, proto, canonname, sa = addrs[0] key = client_key(r_addr, af) client = self._cache.get(key, None) + if not client: + client = self._cache_dns_client.get(key, None) if not client: if self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: @@ -1075,8 +1079,18 @@ def _handle_server(self): return client = socket.socket(af, socktype, proto) client.setblocking(False) - self._cache[key] = client - self._client_fd_to_server_addr[client.fileno()] = r_addr + is_dns = False + if len(data) > 12 and data[11:19] == b"\x00\x01\x00\x00\x00\x00\x00\x00": + is_dns = True + else: + pass + #logging.info("unknown data %s" % (binascii.hexlify(data),)) + if sa[1] == 53 and is_dns: #DNS + logging.debug("DNS query %s from %s:%d" % (common.to_str(sa[0]), r_addr[0], r_addr[1])) + self._cache_dns_client[key] = client + else: + self._cache[key] = client + self._client_fd_to_server_addr[client.fileno()] = (r_addr, af) self._sockets.add(client.fileno()) self._eventloop.add(client, eventloop.POLL_IN, self) @@ -1088,6 +1102,7 @@ def _handle_server(self): r_addr[0], r_addr[1])) self._cache.clear(self._udp_cache_size) + self._cache_dns_client.clear(16) if self._is_local: ref_iv = [encrypt.encrypt_new_iv(self._method)] @@ -1215,7 +1230,13 @@ def _handle_client(self, sock): client_addr = self._client_fd_to_server_addr.get(sock.fileno()) if client_addr: self.server_transfer_dl += len(response) - self.write_to_server_socket(response, client_addr) + self.write_to_server_socket(response, client_addr[0]) + key = client_key(client_addr[0], client_addr[1]) + client = self._cache_dns_client.get(key, None) + if client: + logging.debug("remove dns client %s:%d" % (client_addr[0][0], client_addr[0][1])) + del self._cache_dns_client[key] + self._close_client(client) else: # this packet is from somewhere else we know # simply drop that packet @@ -1344,6 +1365,7 @@ def handle_event(self, sock, fd, event): def handle_periodic(self): if self._closed: self._cache.clear(0) + self._cache_dns_client.clear(0) self._dns_cache.sweep() if self._eventloop: self._eventloop.remove_periodic(self.handle_periodic) @@ -1355,6 +1377,7 @@ def handle_periodic(self): else: before_sweep_size = len(self._sockets) self._cache.sweep() + self._cache_dns_client.sweep() self._dns_cache.sweep() if before_sweep_size != len(self._sockets): logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) @@ -1369,3 +1392,4 @@ def close(self, next_tick=False): self._eventloop.remove(self._server_socket) self._server_socket.close() self._cache.clear(0) + self._cache_dns_client.clear(0) From 2e65becc44e877ef4924102749cceb402e3cd38b Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 23 May 2016 17:28:37 +0800 Subject: [PATCH 176/487] fix data size --- shadowsocks/udprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 1b119ea9..347c89b0 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1080,7 +1080,7 @@ def _handle_server(self): client = socket.socket(af, socktype, proto) client.setblocking(False) is_dns = False - if len(data) > 12 and data[11:19] == b"\x00\x01\x00\x00\x00\x00\x00\x00": + if len(data) > 20 and data[11:19] == b"\x00\x01\x00\x00\x00\x00\x00\x00": is_dns = True else: pass From 8b35508b6fff683934d6118a2b62b0a146f4e92e Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 23 May 2016 17:44:01 +0800 Subject: [PATCH 177/487] show exception peer name --- shadowsocks/eventloop.py | 5 +++++ shadowsocks/tcprelay.py | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index ce9c11bc..b20ccf8d 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -216,6 +216,11 @@ def run(self): handler.handle_event(sock, fd, event) except (OSError, IOError) as e: shell.print_exception(e) + try: + addr = sock.getpeername()[:2] + logging.error('exception peer name %s:%d' % (addr[0], addr[1])) + except: + pass now = time.time() if asap or now - self._last_time >= TIMEOUT_PRECISION: for callback in self._periodic_callbacks: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 4a456a33..0fedd80d 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1008,9 +1008,6 @@ def handle_event(self, sock, fd, event): if error_no in (errno.EAGAIN, errno.EINPROGRESS, errno.EWOULDBLOCK): return - elif error_no == errno.ECONNRESET: - shell.print_exception(e) - logging.info("recv RST, ignore") else: shell.print_exception(e) if self._config['verbose']: From b3ab5d7c4c6d033b042d10ebc6f972fa3a6cebf1 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 24 May 2016 02:21:00 +0800 Subject: [PATCH 178/487] show exception peer name --- shadowsocks/tcprelay.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 0fedd80d..866cda95 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -768,12 +768,22 @@ def _on_local_error(self): logging.debug('got local error') if self._local_sock: logging.error(eventloop.get_sock_error(self._local_sock)) + try: + addr = self._local_sock.getpeername()[:2] + logging.error('local exception peer name %s:%d' % (addr[0], addr[1])) + except: + pass self.destroy() def _on_remote_error(self): logging.debug('got remote error') if self._remote_sock: logging.error(eventloop.get_sock_error(self._remote_sock)) + try: + addr = self._remote_sock.getpeername()[:2] + logging.error('remote exception peer name %s:%d' % (addr[0], addr[1])) + except: + pass self.destroy() def handle_event(self, sock, event): From 3c64e849b0e1df573c9858b977b0381031336a34 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 24 May 2016 14:52:38 +0800 Subject: [PATCH 179/487] log client address --- shadowsocks/eventloop.py | 2 +- shadowsocks/tcprelay.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index b20ccf8d..746e728d 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -220,7 +220,7 @@ def run(self): addr = sock.getpeername()[:2] logging.error('exception peer name %s:%d' % (addr[0], addr[1])) except: - pass + logging.error('no peer name') now = time.time() if asap or now - self._last_time >= TIMEOUT_PRECISION: for callback in self._periodic_callbacks: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 866cda95..c0afadb9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -768,22 +768,24 @@ def _on_local_error(self): logging.debug('got local error') if self._local_sock: logging.error(eventloop.get_sock_error(self._local_sock)) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) try: - addr = self._local_sock.getpeername()[:2] + addr = self._local_sock.getpeername() logging.error('local exception peer name %s:%d' % (addr[0], addr[1])) except: - pass + logging.error('no peer name') self.destroy() def _on_remote_error(self): logging.debug('got remote error') if self._remote_sock: logging.error(eventloop.get_sock_error(self._remote_sock)) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) try: - addr = self._remote_sock.getpeername()[:2] + addr = self._remote_sock.getpeername() logging.error('remote exception peer name %s:%d' % (addr[0], addr[1])) except: - pass + logging.error('no peer name') self.destroy() def handle_event(self, sock, event): From ce189ecd85b8de9fffce2b05846483b75879566f Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 25 May 2016 22:21:17 +0800 Subject: [PATCH 180/487] remove useless log add openssl rand_bytes --- shadowsocks/crypto/openssl.py | 12 ++++++++++++ shadowsocks/encrypt.py | 6 ++++-- shadowsocks/eventloop.py | 5 ----- shadowsocks/tcprelay.py | 15 ++++----------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/shadowsocks/crypto/openssl.py b/shadowsocks/crypto/openssl.py index 3775b6c4..7a4df2b1 100644 --- a/shadowsocks/crypto/openssl.py +++ b/shadowsocks/crypto/openssl.py @@ -51,6 +51,10 @@ def load_openssl(): libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,) + + libcrypto.RAND_bytes.restype = c_int + libcrypto.RAND_bytes.argtypes = (c_void_p, c_int) + if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'): libcrypto.OpenSSL_add_all_ciphers() @@ -68,6 +72,14 @@ def load_cipher(cipher_name): return cipher() return None +def rand_bytes(length): + if not loaded: + load_openssl() + buf = create_string_buffer(length) + r = libcrypto.RAND_bytes(buf, length) + if r <= 0: + raise Exception('RAND_bytes return error') + return buf.raw class OpenSSLCrypto(object): def __init__(self, cipher_name, key, iv, op): diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index c3fc0690..e8f51b74 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -34,8 +34,10 @@ def random_string(length): - return os.urandom(length) - + try: + return os.urandom(length) + except NotImplementedError as e: + return openssl.rand_bytes(length) cached_keys = {} diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 746e728d..ce9c11bc 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -216,11 +216,6 @@ def run(self): handler.handle_event(sock, fd, event) except (OSError, IOError) as e: shell.print_exception(e) - try: - addr = sock.getpeername()[:2] - logging.error('exception peer name %s:%d' % (addr[0], addr[1])) - except: - logging.error('no peer name') now = time.time() if asap or now - self._last_time >= TIMEOUT_PRECISION: for callback in self._periodic_callbacks: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c0afadb9..a58576a7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -769,23 +769,16 @@ def _on_local_error(self): if self._local_sock: logging.error(eventloop.get_sock_error(self._local_sock)) logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) - try: - addr = self._local_sock.getpeername() - logging.error('local exception peer name %s:%d' % (addr[0], addr[1])) - except: - logging.error('no peer name') self.destroy() def _on_remote_error(self): logging.debug('got remote error') if self._remote_sock: logging.error(eventloop.get_sock_error(self._remote_sock)) - logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) - try: - addr = self._remote_sock.getpeername() - logging.error('remote exception peer name %s:%d' % (addr[0], addr[1])) - except: - logging.error('no peer name') + if self._remote_address: + logging.error("when connect to %s:%d from %s:%d" % (self._remote_address[0], self._remote_address[1], self._client_address[0], self._client_address[1])) + else: + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() def handle_event(self, sock, event): From 7ccee594d5f8fd67c97e3acbe9925ce80a7cfe23 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 29 May 2016 15:32:22 +0800 Subject: [PATCH 181/487] catch sys exception & close port --- shadowsocks/tcprelay.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a58576a7..e8709b98 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -108,8 +108,13 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, # if is_local, this is sslocal self._is_local = is_local self._stage = STAGE_INIT - self._encryptor = encrypt.Encryptor(config['password'], - config['method']) + try: + self._encryptor = encrypt.Encryptor(config['password'], + config['method']) + except Exception: + self._stage = STAGE_DESTROYED + logging.error('creater encryptor fail at port %d', server._listen_port) + return self._encrypt_correct = True self._obfs = obfs.obfs(config['obfs']) server_info = obfs.server_info(server.obfs_data) @@ -149,6 +154,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._upstream_status = WAIT_STATUS_READING self._downstream_status = WAIT_STATUS_INIT self._client_address = local_sock.getpeername()[:2] + self._accept_address = local_sock.getsockname()[:2] self._remote_address = None if 'forbidden_ip' in config: self._forbidden_iplist = config['forbidden_ip'] @@ -530,7 +536,7 @@ def _create_remote_socket(self, ip, port): elif self._bindv6 and af == socket.AF_INET6: bind_addr = self._bindv6 else: - bind_addr = self._local_sock.getsockname()[0] + bind_addr = self._accept_address[0] bind_addr = bind_addr.replace("::ffff:", "") if bind_addr in self._ignore_bind_list: @@ -816,6 +822,9 @@ def _log_error(self, e): logging.error('%s when handling connection from %s:%d' % (e, self._client_address[0], self._client_address[1])) + def stage(self): + return self._stage + def destroy(self): # destroy the handler and release any resources # promises: @@ -1005,9 +1014,11 @@ def handle_event(self, sock, fd, event): try: logging.debug('accept') conn = self._server_socket.accept() - TCPRelayHandler(self, self._fd_to_handlers, + handler = TCPRelayHandler(self, self._fd_to_handlers, self._eventloop, conn[0], self._config, self._dns_resolver, self._is_local) + if handler.stage() == STAGE_DESTROYED: + conn[0].close() except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) if error_no in (errno.EAGAIN, errno.EINPROGRESS, From 9eefd27152a6d98757143cc73d47fc79ca6b9956 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 30 May 2016 01:39:42 +0800 Subject: [PATCH 182/487] add file name and line number in log --- shadowsocks/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index babac982..ed0cc42e 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -283,7 +283,7 @@ def get_config(is_local): level = logging.INFO verbose = config['verbose'] logging.basicConfig(level=level, - format='%(asctime)s %(levelname)-8s %(message)s', + format='%(asctime)s %(levelname)-8s %(filename)s:%(lineno)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') check_config(config, is_local) From 82ffa15a562e68a6797be679c263ed4de175690d Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 1 Jun 2016 15:40:28 +0800 Subject: [PATCH 183/487] add connection stat log --- server_pool.py | 5 +++-- shadowsocks/manager.py | 4 ++-- shadowsocks/server.py | 12 +++++++---- shadowsocks/tcprelay.py | 46 +++++++++++++++++++++++++++++++++++++++-- shadowsocks/udprelay.py | 2 +- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/server_pool.py b/server_pool.py index 27e1168e..cf200e16 100644 --- a/server_pool.py +++ b/server_pool.py @@ -61,6 +61,7 @@ def __init__(self): self.tcp_ipv6_servers_pool = {} self.udp_servers_pool = {} self.udp_ipv6_servers_pool = {} + self.stat_counter = {} self.loop = eventloop.EventLoop() thread = MainThread( (self.loop, self.dns_resolver, self.mgr) ) @@ -129,11 +130,11 @@ def new_server(self, port, password): try: logging.info("starting server at [%s]:%d" % (a_config['server'], port)) - tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) + tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False, stat_counter=self.stat_counter) tcp_server.add_to_loop(self.loop) self.tcp_ipv6_servers_pool.update({port: tcp_server}) - udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False) + udp_server = udprelay.UDPRelay(a_config, self.dns_resolver, False, stat_counter=self.stat_counter) udp_server.add_to_loop(self.loop) self.udp_ipv6_servers_pool.update({port: udp_server}) diff --git a/shadowsocks/manager.py b/shadowsocks/manager.py index 57b95b70..80d0a320 100644 --- a/shadowsocks/manager.py +++ b/shadowsocks/manager.py @@ -86,9 +86,9 @@ def add_port(self, config): return logging.info("adding server at %s:%d" % (config['server'], port)) t = tcprelay.TCPRelay(config, self._dns_resolver, False, - self.stat_callback) + stat_callback=self.stat_callback) u = udprelay.UDPRelay(config, self._dns_resolver, False, - self.stat_callback) + stat_callback=self.stat_callback) t.add_to_loop(self._loop) u.add_to_loop(self._loop) self._relays[port] = (t, u) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 3547309e..f8deeeb6 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -61,6 +61,10 @@ def main(): tcp_servers = [] udp_servers = [] dns_resolver = asyncdns.DNSResolver() + if int(config['workers']) > 1: + stat_counter_dict = None + else: + stat_counter_dict = {} port_password = config['port_password'] config_password = config.get('password', 'm') del config['port_password'] @@ -108,8 +112,8 @@ def main(): a_config['server'] = a_config['server_ipv6'] logging.info("starting server at [%s]:%d" % (a_config['server'], int(port))) - tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) - udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) + tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False, stat_counter=stat_counter_dict)) + udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False, stat_counter=stat_counter_dict)) if a_config['server_ipv6'] == b"::": ipv6_ok = True except Exception as e: @@ -128,8 +132,8 @@ def main(): a_config['out_bindv6'] = bindv6 logging.info("starting server at %s:%d" % (a_config['server'], int(port))) - tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) - udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) + tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False, stat_counter=stat_counter_dict)) + udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False, stat_counter=stat_counter_dict)) except Exception as e: if not ipv6_ok: shell.print_exception(e) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e8709b98..4fb0dd79 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -170,6 +170,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self.last_activity = 0 self._update_activity() self._server.add_connection(1) + self._server.stat_add(common.to_str(self._client_address[0]), 1) def __hash__(self): # default __hash__ is id / 16 @@ -877,9 +878,10 @@ def destroy(self): self._dns_resolver.remove_callback(self._handle_dns_resolved) self._server.remove_handler(self) self._server.add_connection(-1) + self._server.stat_add(common.to_str(self._client_address[0]), -1) class TCPRelay(object): - def __init__(self, config, dns_resolver, is_local, stat_callback=None): + def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_counter=None): self._config = config self._is_local = is_local self._dns_resolver = dns_resolver @@ -925,8 +927,9 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None): except socket.error: logging.error('warning: fast open is not available') self._config['fast_open'] = False - server_socket.listen(1024) + server_socket.listen(config.get('max_connect', 1024)) self._server_socket = server_socket + self._stat_counter = stat_counter self._stat_callback = stat_callback def add_to_loop(self, loop): @@ -950,6 +953,45 @@ def add_connection(self, val): self.server_connections += val logging.debug('server port %5d connections = %d' % (self._listen_port, self.server_connections,)) + def update_stat(self, port, stat_dict, val): + newval = stat_dict.get(0, 0) + val + stat_dict[0] = newval + logging.debug('port %d connections %d' % (port, newval)) + connections_step = 25 + if newval >= stat_dict.get(-1, 0) + connections_step: + logging.info('port %d connections up to %d' % (port, newval)) + stat_dict[-1] = stat_dict.get(-1, 0) + connections_step + elif newval <= stat_dict.get(-1, 0) - connections_step: + logging.info('port %d connections down to %d' % (port, newval)) + stat_dict[-1] = stat_dict.get(-1, 0) - connections_step + + def stat_add(self, local_addr, val): + if self._stat_counter is not None: + if self._listen_port not in self._stat_counter: + self._stat_counter[self._listen_port] = {} + newval = self._stat_counter[self._listen_port].get(local_addr, 0) + val + logging.debug('port %d addr %s connections %d' % (self._listen_port, local_addr, newval)) + if newval <= 0: + if local_addr in self._stat_counter[self._listen_port]: + del self._stat_counter[self._listen_port][local_addr] + if len(self._stat_counter[self._listen_port]) == 0: + del self._stat_counter[self._listen_port] + else: + self._stat_counter[self._listen_port][local_addr] = newval + self.update_stat(self._listen_port, self._stat_counter[self._listen_port], val) + + newval = self._stat_counter.get(0, 0) + val + self._stat_counter[0] = newval + logging.debug('Total connections %d' % newval) + + connections_step = 50 + if newval >= self._stat_counter.get(-1, 0) + connections_step: + logging.info('Total connections up to %d' % newval) + self._stat_counter[-1] = self._stat_counter.get(-1, 0) + connections_step + elif newval <= self._stat_counter.get(-1, 0) - connections_step: + logging.info('Total connections down to %d' % newval) + self._stat_counter[-1] = self._stat_counter.get(-1, 0) - connections_step + def update_activity(self, handler, data_len): if data_len and self._stat_callback: self._stat_callback(self._listen_port, data_len) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 347c89b0..3781d7db 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -863,7 +863,7 @@ def client_key(source_addr, server_af): class UDPRelay(object): - def __init__(self, config, dns_resolver, is_local, stat_callback=None): + def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_counter=None): self._config = config if config.get('connect_verbose_info', 0) > 0: common.connect_log = logging.info From 91576d1e08b1e02a632f1f6a1bf5fd58eff3521c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 3 Jun 2016 20:56:59 +0800 Subject: [PATCH 184/487] detect thread status add API params --- Config.py | 16 ++++++++++++++-- db_transfer.py | 18 +++++++++--------- server.py | 4 ++-- server_pool.py | 4 ++-- shadowsocks/tcprelay.py | 4 ++-- 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/Config.py b/Config.py index 752ebde8..f98f2041 100644 --- a/Config.py +++ b/Config.py @@ -1,12 +1,24 @@ -#Config +# Config +TRANSFER_MUL = 1.0 +API_INTERFACE = 'mysql' + +# Mysql MYSQL_HOST = 'mdss.mengsky.net' MYSQL_PORT = 3306 MYSQL_USER = 'ss' MYSQL_PASS = 'ss' MYSQL_DB = 'shadowsocks' -MYSQL_TRANSFER_MUL = 1.0 MYSQL_UPDATE_TIME = 60 +# API +API_URL = 'breakwa11.org' +API_PORT = 80 +API_PATH = '/mu/v2/' +API_TOKEN = 'abcdef' +API_NODE_ID = 'id001' +API_UPDATE_TIME = 60 + +# Manager (ignore this) MANAGE_PASS = 'ss233333333' #if you want manage in other server you should set this value to global ip MANAGE_BIND_IP = '127.0.0.1' diff --git a/db_transfer.py b/db_transfer.py index d5c6994f..8b46503a 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -39,16 +39,16 @@ def push_db_all_user(self): continue elif last_transfer[id][0] <= curr_transfer[id][0] and \ last_transfer[id][1] <= curr_transfer[id][1]: - dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * Config.MYSQL_TRANSFER_MUL), - int((curr_transfer[id][1] - last_transfer[id][1]) * Config.MYSQL_TRANSFER_MUL)] + dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * Config.TRANSFER_MUL), + int((curr_transfer[id][1] - last_transfer[id][1]) * Config.TRANSFER_MUL)] else: - dt_transfer[id] = [int(curr_transfer[id][0] * Config.MYSQL_TRANSFER_MUL), - int(curr_transfer[id][1] * Config.MYSQL_TRANSFER_MUL)] + dt_transfer[id] = [int(curr_transfer[id][0] * Config.TRANSFER_MUL), + int(curr_transfer[id][1] * Config.TRANSFER_MUL)] else: if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: continue - dt_transfer[id] = [int(curr_transfer[id][0] * Config.MYSQL_TRANSFER_MUL), - int(curr_transfer[id][1] * Config.MYSQL_TRANSFER_MUL)] + dt_transfer[id] = [int(curr_transfer[id][0] * Config.TRANSFER_MUL), + int(curr_transfer[id][1] * Config.TRANSFER_MUL)] query_head = 'UPDATE user' query_sub_when = '' @@ -166,10 +166,10 @@ def del_server_out_of_bound_safe(last_rows, rows): def del_servers(): for port in ServerPool.get_instance().tcp_servers_pool.keys(): if ServerPool.get_instance().server_is_run(port) > 0: - ServerPool.get_instance().cb_del_server(port) + ServerPool.get_instance().cb_del_server(port) for port in ServerPool.get_instance().tcp_ipv6_servers_pool.keys(): if ServerPool.get_instance().server_is_run(port) > 0: - ServerPool.get_instance().cb_del_server(port) + ServerPool.get_instance().cb_del_server(port) @staticmethod def thread_db(): @@ -190,7 +190,7 @@ def thread_db(): trace = traceback.format_exc() logging.error(trace) #logging.warn('db thread except:%s' % e) - if DbTransfer.get_instance().event.wait(Config.MYSQL_UPDATE_TIME): + if DbTransfer.get_instance().event.wait(Config.MYSQL_UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive(): break except KeyboardInterrupt as e: pass diff --git a/server.py b/server.py index a28f026d..db375ad4 100644 --- a/server.py +++ b/server.py @@ -50,8 +50,8 @@ def main(): thread = MainThread() thread.start() try: - while True: - time.sleep(99999) + while thread.is_alive(): + time.sleep(10) except (KeyboardInterrupt, IOError, OSError) as e: import traceback traceback.print_exc() diff --git a/server_pool.py b/server_pool.py index cf200e16..bcfb7316 100644 --- a/server_pool.py +++ b/server_pool.py @@ -64,8 +64,8 @@ def __init__(self): self.stat_counter = {} self.loop = eventloop.EventLoop() - thread = MainThread( (self.loop, self.dns_resolver, self.mgr) ) - thread.start() + self.thread = MainThread( (self.loop, self.dns_resolver, self.mgr) ) + self.thread.start() @staticmethod def get_instance(): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 4fb0dd79..77331d89 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -170,7 +170,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self.last_activity = 0 self._update_activity() self._server.add_connection(1) - self._server.stat_add(common.to_str(self._client_address[0]), 1) + self._server.stat_add(self._client_address[0], 1) def __hash__(self): # default __hash__ is id / 16 @@ -878,7 +878,7 @@ def destroy(self): self._dns_resolver.remove_callback(self._handle_dns_resolved) self._server.remove_handler(self) self._server.add_connection(-1) - self._server.stat_add(common.to_str(self._client_address[0]), -1) + self._server.stat_add(self._client_address[0], -1) class TCPRelay(object): def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_counter=None): From 6932548d8e642450f04822b25b42c7168080921e Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 3 Jun 2016 21:18:50 +0800 Subject: [PATCH 185/487] ulimit -n 512000 --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 8fceb971..82a2105f 100644 --- a/run.sh +++ b/run.sh @@ -1,6 +1,6 @@ #!/bin/bash cd `dirname $0` eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') -ulimit -n 131072 +ulimit -n 512000 nohup python server.py m>> ssserver.log 2>&1 & From 4d43e12139e94fa194a23b352a7b2d1981fa4858 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 3 Jun 2016 22:15:59 +0800 Subject: [PATCH 186/487] rename API_HOST --- Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Config.py b/Config.py index f98f2041..d3f21208 100644 --- a/Config.py +++ b/Config.py @@ -11,7 +11,7 @@ MYSQL_UPDATE_TIME = 60 # API -API_URL = 'breakwa11.org' +API_HOST = 'breakwa11.org' API_PORT = 80 API_PATH = '/mu/v2/' API_TOKEN = 'abcdef' From 569aaa5248d37f512b46cc11323dee17c17044cc Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 8 Jun 2016 16:15:26 +0800 Subject: [PATCH 187/487] hot load config support userapiconfig --- Config.py => apiconfig.py | 6 +++--- config.json | 1 - configloader.py | 27 +++++++++++++++++++++++++++ db_transfer.py | 26 +++++++++++++------------- logrun.sh | 6 ++++++ run.sh | 2 +- server_pool.py | 13 +++++-------- shadowsocks/logrun.sh | 6 ++++++ shadowsocks/run.sh | 3 ++- 9 files changed, 63 insertions(+), 27 deletions(-) rename Config.py => apiconfig.py (84%) create mode 100644 configloader.py create mode 100644 logrun.sh create mode 100644 shadowsocks/logrun.sh diff --git a/Config.py b/apiconfig.py similarity index 84% rename from Config.py rename to apiconfig.py index d3f21208..580e55b3 100644 --- a/Config.py +++ b/apiconfig.py @@ -1,6 +1,7 @@ # Config TRANSFER_MUL = 1.0 -API_INTERFACE = 'mysql' +NODE_ID = 1 +API_INTERFACE = 'sspanelv2' #sspanelv2, muapiv2 # Mysql MYSQL_HOST = 'mdss.mengsky.net' @@ -11,11 +12,10 @@ MYSQL_UPDATE_TIME = 60 # API -API_HOST = 'breakwa11.org' +API_HOST = 'breakwa11.moe' API_PORT = 80 API_PATH = '/mu/v2/' API_TOKEN = 'abcdef' -API_NODE_ID = 'id001' API_UPDATE_TIME = 60 # Manager (ignore this) diff --git a/config.json b/config.json index 30bc51da..8f7a4c93 100644 --- a/config.json +++ b/config.json @@ -16,5 +16,4 @@ "connect_verbose_info": 0, "redirect": "", "fast_open": false, - "workers": 1 } diff --git a/configloader.py b/configloader.py new file mode 100644 index 00000000..8d0ec0d7 --- /dev/null +++ b/configloader.py @@ -0,0 +1,27 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +config = None + +def load_config(): + global config + try: + import userapiconfig + reload(userapiconfig) + config = userapiconfig + return + except: + pass + try: + import apiconfig + reload(apiconfig) + config = apiconfig + except: + pass + +def get_config(): + global config + return config + +load_config() + diff --git a/db_transfer.py b/db_transfer.py index 8b46503a..b64eb65b 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -6,9 +6,9 @@ import time import sys from server_pool import ServerPool -import Config import traceback from shadowsocks import common +from configloader import load_config, get_config class DbTransfer(object): @@ -39,16 +39,16 @@ def push_db_all_user(self): continue elif last_transfer[id][0] <= curr_transfer[id][0] and \ last_transfer[id][1] <= curr_transfer[id][1]: - dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * Config.TRANSFER_MUL), - int((curr_transfer[id][1] - last_transfer[id][1]) * Config.TRANSFER_MUL)] + dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * get_config().TRANSFER_MUL), + int((curr_transfer[id][1] - last_transfer[id][1]) * get_config().TRANSFER_MUL)] else: - dt_transfer[id] = [int(curr_transfer[id][0] * Config.TRANSFER_MUL), - int(curr_transfer[id][1] * Config.TRANSFER_MUL)] + dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), + int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] else: if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: continue - dt_transfer[id] = [int(curr_transfer[id][0] * Config.TRANSFER_MUL), - int(curr_transfer[id][1] * Config.TRANSFER_MUL)] + dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), + int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] query_head = 'UPDATE user' query_sub_when = '' @@ -71,8 +71,8 @@ def push_db_all_user(self): ' END, t = ' + str(int(last_time)) + \ ' WHERE port IN (%s)' % query_sub_in #print query_sql - conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, - passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, + passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') cur = conn.cursor() cur.execute(query_sql) cur.close() @@ -90,8 +90,8 @@ def pull_db_all_user(): except Exception as e: keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] reload(cymysql) - conn = cymysql.connect(host=Config.MYSQL_HOST, port=Config.MYSQL_PORT, user=Config.MYSQL_USER, - passwd=Config.MYSQL_PASS, db=Config.MYSQL_DB, charset='utf8') + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, + passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') cur = conn.cursor() cur.execute("SELECT " + ','.join(keys) + " FROM user") rows = [] @@ -180,7 +180,7 @@ def thread_db(): last_rows = [] try: while True: - reload(Config) + load_config() try: DbTransfer.get_instance().push_db_all_user() rows = DbTransfer.get_instance().pull_db_all_user() @@ -190,7 +190,7 @@ def thread_db(): trace = traceback.format_exc() logging.error(trace) #logging.warn('db thread except:%s' % e) - if DbTransfer.get_instance().event.wait(Config.MYSQL_UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive(): + if DbTransfer.get_instance().event.wait(get_config().MYSQL_UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive(): break except KeyboardInterrupt as e: pass diff --git a/logrun.sh b/logrun.sh new file mode 100644 index 00000000..82a2105f --- /dev/null +++ b/logrun.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd `dirname $0` +eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') +ulimit -n 512000 +nohup python server.py m>> ssserver.log 2>&1 & + diff --git a/run.sh b/run.sh index 82a2105f..89f7b05b 100644 --- a/run.sh +++ b/run.sh @@ -2,5 +2,5 @@ cd `dirname $0` eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') ulimit -n 512000 -nohup python server.py m>> ssserver.log 2>&1 & +nohup python server.py m>> /dev/null 2>&1 & diff --git a/server_pool.py b/server_pool.py index bcfb7316..b897d20d 100644 --- a/server_pool.py +++ b/server_pool.py @@ -24,16 +24,11 @@ import os import logging import time -from shadowsocks import shell -from shadowsocks import eventloop -from shadowsocks import tcprelay -from shadowsocks import udprelay -from shadowsocks import asyncdns +from shadowsocks import shell, eventloop, tcprelay, udprelay, asyncdns import threading import sys -import asyncmgr -import Config from socket import * +from configloader import load_config, get_config class MainThread(threading.Thread): def __init__(self, params): @@ -127,6 +122,7 @@ def new_server(self, port, password): a_config['server'] = a_config['server_ipv6'] a_config['server_port'] = port a_config['password'] = password + a_config['max_connect'] = 128 try: logging.info("starting server at [%s]:%d" % (a_config['server'], port)) @@ -151,6 +147,7 @@ def new_server(self, port, password): a_config = self.config.copy() a_config['server_port'] = port a_config['password'] = password + a_config['max_connect'] = 128 try: logging.info("starting server at %s:%d" % (a_config['server'], port)) @@ -173,7 +170,7 @@ def del_server(self, port): logging.info("del server at %d" % port) try: udpsock = socket(AF_INET, SOCK_DGRAM) - udpsock.sendto('%s:%s:0:0' % (Config.MANAGE_PASS, port), (Config.MANAGE_BIND_IP, Config.MANAGE_PORT)) + udpsock.sendto('%s:%s:0:0' % (get_config().MANAGE_PASS, port), (get_config().MANAGE_BIND_IP, get_config().MANAGE_PORT)) udpsock.close() except Exception as e: logging.warn(e) diff --git a/shadowsocks/logrun.sh b/shadowsocks/logrun.sh new file mode 100644 index 00000000..2b7a5bd8 --- /dev/null +++ b/shadowsocks/logrun.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd `dirname $0` +eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') +ulimit -n 4096 +nohup python server.py a >> ssserver.log 2>&1 & + diff --git a/shadowsocks/run.sh b/shadowsocks/run.sh index 497ceb84..7a91779c 100644 --- a/shadowsocks/run.sh +++ b/shadowsocks/run.sh @@ -1,5 +1,6 @@ #!/bin/bash cd `dirname $0` eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') -nohup python server.py a >> ssserver.log 2>&1 & +ulimit -n 4096 +nohup python server.py a >> /dev/null 2>&1 & From 51bd3fcea0feb55600ddaffd857a963ddc572ab0 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 8 Jun 2016 21:18:10 +0800 Subject: [PATCH 188/487] fix config.json --- config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json b/config.json index 8f7a4c93..0a72c704 100644 --- a/config.json +++ b/config.json @@ -15,5 +15,5 @@ "dns_ipv6": false, "connect_verbose_info": 0, "redirect": "", - "fast_open": false, + "fast_open": false } From c7864acc379214e32da294dd8d6c237022a4227e Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 12 Jun 2016 17:41:05 +0800 Subject: [PATCH 189/487] add sspanelv3 interface, read the 'method' row --- apiconfig.py | 2 +- db_transfer.py | 12 ++++++++---- server_pool.py | 4 +++- switchrule.py | 10 ++++++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/apiconfig.py b/apiconfig.py index 580e55b3..4713db5f 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,7 +1,7 @@ # Config TRANSFER_MUL = 1.0 NODE_ID = 1 -API_INTERFACE = 'sspanelv2' #sspanelv2, muapiv2 +API_INTERFACE = 'sspanelv2' #sspanelv2, sspanelv3, muapiv2 # Mysql MYSQL_HOST = 'mdss.mengsky.net' diff --git a/db_transfer.py b/db_transfer.py index b64eb65b..5d315eb7 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -124,6 +124,10 @@ def del_server_out_of_bound_safe(last_rows, rows): port = row['port'] passwd = common.to_bytes(row['passwd']) + cfg = {} + for name in ['method', 'obfs', 'protocol']: + if name in row: + cfg[name] = row[name] if port not in cur_servers: cur_servers[port] = passwd @@ -140,12 +144,12 @@ def del_server_out_of_bound_safe(last_rows, rows): #password changed logging.info('db stop server at port [%s] reason: password changed' % (port,)) ServerPool.get_instance().cb_del_server(port) - new_servers[port] = passwd + new_servers[port] = (passwd, cfg) elif allow and ServerPool.get_instance().server_run_status(port) is False: #new_servers[port] = passwd logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) - ServerPool.get_instance().new_server(port, passwd) + ServerPool.get_instance().new_server(port, passwd, cfg) for row in last_rows: if row['port'] in cur_servers: @@ -158,9 +162,9 @@ def del_server_out_of_bound_safe(last_rows, rows): from shadowsocks import eventloop DbTransfer.get_instance().event.wait(eventloop.TIMEOUT_PRECISION) for port in new_servers.keys(): - passwd = new_servers[port] + passwd, cfg = new_servers[port] logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) - ServerPool.get_instance().new_server(port, passwd) + ServerPool.get_instance().new_server(port, passwd, cfg) @staticmethod def del_servers(): diff --git a/server_pool.py b/server_pool.py index b897d20d..f0529a57 100644 --- a/server_pool.py +++ b/server_pool.py @@ -106,7 +106,7 @@ def server_run_status(self, port): return False return True - def new_server(self, port, password): + def new_server(self, port, password, user_config): ret = True port = int(port) ipv6_ok = False @@ -117,6 +117,7 @@ def new_server(self, port, password): return 'this port server is already running' else: a_config = self.config.copy() + a_config.update(user_config) if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": a_config['server_ipv6'] = a_config['server_ipv6'][1:-1] a_config['server'] = a_config['server_ipv6'] @@ -145,6 +146,7 @@ def new_server(self, port, password): return 'this port server is already running' else: a_config = self.config.copy() + a_config.update(user_config) a_config['server_port'] = port a_config['password'] = password a_config['max_connect'] = 128 diff --git a/switchrule.py b/switchrule.py index 3721d2c9..5785850b 100644 --- a/switchrule.py +++ b/switchrule.py @@ -1,6 +1,12 @@ +from configloader import load_config, get_config + def getKeys(): - return ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] - #return ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable', 'plan' ] # append the column name 'plan' + load_config() + key_list = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] + if get_config().API_INTERFACE == 'sspanelv3': + key_list += ['method'] + return key_list + #return key_list + ['plan'] # append the column name 'plan' def isTurnOn(row): return True From 7f26f43f883fa8a0791a38ba71fc01e193902fb1 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 12 Jun 2016 18:48:07 +0800 Subject: [PATCH 190/487] allow custom obfs & protocol --- apiconfig.py | 2 +- switchrule.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apiconfig.py b/apiconfig.py index 4713db5f..c65577a7 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,7 +1,7 @@ # Config TRANSFER_MUL = 1.0 NODE_ID = 1 -API_INTERFACE = 'sspanelv2' #sspanelv2, sspanelv3, muapiv2 +API_INTERFACE = 'sspanelv2' #sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) # Mysql MYSQL_HOST = 'mdss.mengsky.net' diff --git a/switchrule.py b/switchrule.py index 5785850b..e102c634 100644 --- a/switchrule.py +++ b/switchrule.py @@ -5,6 +5,8 @@ def getKeys(): key_list = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] if get_config().API_INTERFACE == 'sspanelv3': key_list += ['method'] + elif get_config().API_INTERFACE == 'sspanelv3ssr': + key_list += ['method', 'obfs', 'protocol'] return key_list #return key_list + ['plan'] # append the column name 'plan' From 4af94729864bea8042139747051c767a9ceb4920 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 12 Jun 2016 22:46:47 +0800 Subject: [PATCH 191/487] detect cfg modification --- db_transfer.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 5d315eb7..b4d596e7 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -124,7 +124,7 @@ def del_server_out_of_bound_safe(last_rows, rows): port = row['port'] passwd = common.to_bytes(row['passwd']) - cfg = {} + cfg = {'password': passwd} for name in ['method', 'obfs', 'protocol']: if name in row: cfg[name] = row[name] @@ -139,12 +139,25 @@ def del_server_out_of_bound_safe(last_rows, rows): if not allow: logging.info('db stop server at port [%s]' % (port,)) ServerPool.get_instance().cb_del_server(port) - elif (port in ServerPool.get_instance().tcp_servers_pool and ServerPool.get_instance().tcp_servers_pool[port]._config['password'] != passwd) \ - or (port in ServerPool.get_instance().tcp_ipv6_servers_pool and ServerPool.get_instance().tcp_ipv6_servers_pool[port]._config['password'] != passwd): - #password changed - logging.info('db stop server at port [%s] reason: password changed' % (port,)) - ServerPool.get_instance().cb_del_server(port) - new_servers[port] = (passwd, cfg) + else: + cfgchange = False + if (port in ServerPool.get_instance().tcp_servers_pool): + relay = ServerPool.get_instance().tcp_servers_pool[port] + for name in ['password', 'method', 'obfs', 'protocol']: + if name in cfg and cfg[name] != relay._config[name]: + cfgchange = True + break; + if (port in ServerPool.get_instance().tcp_ipv6_servers_pool): + relay = ServerPool.get_instance().tcp_ipv6_servers_pool[port] + for name in ['password', 'method', 'obfs', 'protocol']: + if name in cfg and cfg[name] != relay._config[name]: + cfgchange = True + break; + #config changed + if cfgchange: + logging.info('db stop server at port [%s] reason: config changed: %s' % (port, cfg)) + ServerPool.get_instance().cb_del_server(port) + new_servers[port] = (passwd, cfg) elif allow and ServerPool.get_instance().server_run_status(port) is False: #new_servers[port] = passwd From 825aece2193acf95955aa7a40a188883bbdf5786 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 12 Jun 2016 23:06:40 +0800 Subject: [PATCH 192/487] fix char set --- db_transfer.py | 12 ++++++++---- server_pool.py | 4 +--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index b4d596e7..18423e16 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -129,6 +129,10 @@ def del_server_out_of_bound_safe(last_rows, rows): if name in row: cfg[name] = row[name] + for name in cfg.keys(): + if hasattr(cfg[name], 'encode'): + cfg[name] = cfg[name].encode('utf-8') + if port not in cur_servers: cur_servers[port] = passwd else: @@ -141,13 +145,13 @@ def del_server_out_of_bound_safe(last_rows, rows): ServerPool.get_instance().cb_del_server(port) else: cfgchange = False - if (port in ServerPool.get_instance().tcp_servers_pool): + if port in ServerPool.get_instance().tcp_servers_pool: relay = ServerPool.get_instance().tcp_servers_pool[port] for name in ['password', 'method', 'obfs', 'protocol']: if name in cfg and cfg[name] != relay._config[name]: cfgchange = True break; - if (port in ServerPool.get_instance().tcp_ipv6_servers_pool): + if not cfgchange and port in ServerPool.get_instance().tcp_ipv6_servers_pool: relay = ServerPool.get_instance().tcp_ipv6_servers_pool[port] for name in ['password', 'method', 'obfs', 'protocol']: if name in cfg and cfg[name] != relay._config[name]: @@ -162,7 +166,7 @@ def del_server_out_of_bound_safe(last_rows, rows): elif allow and ServerPool.get_instance().server_run_status(port) is False: #new_servers[port] = passwd logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) - ServerPool.get_instance().new_server(port, passwd, cfg) + ServerPool.get_instance().new_server(port, cfg) for row in last_rows: if row['port'] in cur_servers: @@ -177,7 +181,7 @@ def del_server_out_of_bound_safe(last_rows, rows): for port in new_servers.keys(): passwd, cfg = new_servers[port] logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) - ServerPool.get_instance().new_server(port, passwd, cfg) + ServerPool.get_instance().new_server(port, cfg) @staticmethod def del_servers(): diff --git a/server_pool.py b/server_pool.py index f0529a57..63a8f16f 100644 --- a/server_pool.py +++ b/server_pool.py @@ -106,7 +106,7 @@ def server_run_status(self, port): return False return True - def new_server(self, port, password, user_config): + def new_server(self, port, user_config): ret = True port = int(port) ipv6_ok = False @@ -122,7 +122,6 @@ def new_server(self, port, password, user_config): a_config['server_ipv6'] = a_config['server_ipv6'][1:-1] a_config['server'] = a_config['server_ipv6'] a_config['server_port'] = port - a_config['password'] = password a_config['max_connect'] = 128 try: logging.info("starting server at [%s]:%d" % (a_config['server'], port)) @@ -148,7 +147,6 @@ def new_server(self, port, password, user_config): a_config = self.config.copy() a_config.update(user_config) a_config['server_port'] = port - a_config['password'] = password a_config['max_connect'] = 128 try: logging.info("starting server at %s:%d" % (a_config['server'], port)) From 8dac9faa286c68704b44baf28073094d0d7ace35 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 14 Jun 2016 14:06:52 +0800 Subject: [PATCH 193/487] 'obfs', 'protocol', 'method' leave blank that use the config of config.json --- db_transfer.py | 59 ++++++++++++++++++---------------- shadowsocks/obfsplugin/auth.py | 6 ---- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 18423e16..ddb4bcc3 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -2,7 +2,6 @@ # -*- coding: UTF-8 -*- import logging -import cymysql import time import sys from server_pool import ServerPool @@ -25,31 +24,8 @@ def get_instance(): DbTransfer.instance = DbTransfer() return DbTransfer.instance - def push_db_all_user(self): - #更新用户流量到数据库 - last_transfer = self.last_get_transfer - curr_transfer = ServerPool.get_instance().get_servers_transfer() - #上次和本次的增量 - dt_transfer = {} - for id in curr_transfer.keys(): - if id in last_transfer: - if last_transfer[id][0] == curr_transfer[id][0] and last_transfer[id][1] == curr_transfer[id][1]: - continue - elif curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: - continue - elif last_transfer[id][0] <= curr_transfer[id][0] and \ - last_transfer[id][1] <= curr_transfer[id][1]: - dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * get_config().TRANSFER_MUL), - int((curr_transfer[id][1] - last_transfer[id][1]) * get_config().TRANSFER_MUL)] - else: - dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), - int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] - else: - if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: - continue - dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), - int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] - + def update_all_user(self, dt_transfer): + import cymysql query_head = 'UPDATE user' query_sub_when = '' query_sub_when2 = '' @@ -78,10 +54,38 @@ def push_db_all_user(self): cur.close() conn.commit() conn.close() + + def push_db_all_user(self): + #更新用户流量到数据库 + last_transfer = self.last_get_transfer + curr_transfer = ServerPool.get_instance().get_servers_transfer() + #上次和本次的增量 + dt_transfer = {} + for id in curr_transfer.keys(): + if id in last_transfer: + if last_transfer[id][0] == curr_transfer[id][0] and last_transfer[id][1] == curr_transfer[id][1]: + continue + elif curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + continue + elif last_transfer[id][0] <= curr_transfer[id][0] and \ + last_transfer[id][1] <= curr_transfer[id][1]: + dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * get_config().TRANSFER_MUL), + int((curr_transfer[id][1] - last_transfer[id][1]) * get_config().TRANSFER_MUL)] + else: + dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), + int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] + else: + if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + continue + dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), + int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] + + self.update_all_user(dt_transfer) self.last_get_transfer = curr_transfer @staticmethod def pull_db_all_user(): + import cymysql #数据库所有用户信息 try: import switchrule @@ -89,7 +93,6 @@ def pull_db_all_user(): keys = switchrule.getKeys() except Exception as e: keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] - reload(cymysql) conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') cur = conn.cursor() @@ -126,7 +129,7 @@ def del_server_out_of_bound_safe(last_rows, rows): passwd = common.to_bytes(row['passwd']) cfg = {'password': passwd} for name in ['method', 'obfs', 'protocol']: - if name in row: + if name in row and row[name]: cfg[name] = row[name] for name in cfg.keys(): diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 1217cc46..d64c98b5 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -206,8 +206,6 @@ def set_server_info(self, server_info): self.server_info.data.set_max_client(max_client) def pack_data(self, buf): - if len(buf) == 0: - return b'' rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) data = common.chr(len(rnd_data) + 1) + rnd_data + buf data = struct.pack('>H', len(data) + 6) + data @@ -364,8 +362,6 @@ def set_server_info(self, server_info): self.server_info.data.set_max_client(max_client) def pack_data(self, buf): - if len(buf) == 0: - return b'' rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) data = common.chr(len(rnd_data) + 1) + rnd_data + buf data = struct.pack('>H', len(data) + 6) + data @@ -606,8 +602,6 @@ def rnd_data(self, buf_size): return common.chr(255) + struct.pack('>H', len(rnd_data) + 3) + rnd_data def pack_data(self, buf): - if len(buf) == 0: - return b'' data = self.rnd_data(len(buf)) + buf data = struct.pack('>H', len(data) + 6) + data adler32 = zlib.adler32(data) & 0xFFFFFFFF From 9e288c9ac09a6f359e1162c0227428a79a9f320b Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 14 Jun 2016 22:21:23 +0800 Subject: [PATCH 194/487] supper local json database for mu --- apiconfig.py | 6 ++--- db_transfer.py | 71 ++++++++++++++++++++++++++++++++++---------------- mudb.json | 26 ++++++++++++++++++ server.py | 17 ++++++------ switchrule.py | 1 - 5 files changed, 87 insertions(+), 34 deletions(-) create mode 100644 mudb.json diff --git a/apiconfig.py b/apiconfig.py index c65577a7..82415470 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,10 +1,10 @@ # Config TRANSFER_MUL = 1.0 NODE_ID = 1 -API_INTERFACE = 'sspanelv2' #sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) +API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) # Mysql -MYSQL_HOST = 'mdss.mengsky.net' +MYSQL_HOST = '127.0.0.1' MYSQL_PORT = 3306 MYSQL_USER = 'ss' MYSQL_PASS = 'ss' @@ -12,7 +12,7 @@ MYSQL_UPDATE_TIME = 60 # API -API_HOST = 'breakwa11.moe' +API_HOST = '127.0.0.1' API_PORT = 80 API_PATH = '/mu/v2/' API_TOKEN = 'abcdef' diff --git a/db_transfer.py b/db_transfer.py index ddb4bcc3..12e6e559 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -6,24 +6,17 @@ import sys from server_pool import ServerPool import traceback -from shadowsocks import common +from shadowsocks import common, shell from configloader import load_config, get_config -class DbTransfer(object): - - instance = None +db_instance = None +class DbTransfer(object): def __init__(self): import threading self.last_get_transfer = {} self.event = threading.Event() - @staticmethod - def get_instance(): - if DbTransfer.instance is None: - DbTransfer.instance = DbTransfer() - return DbTransfer.instance - def update_all_user(self, dt_transfer): import cymysql query_head = 'UPDATE user' @@ -83,8 +76,7 @@ def push_db_all_user(self): self.update_all_user(dt_transfer) self.last_get_transfer = curr_transfer - @staticmethod - def pull_db_all_user(): + def pull_db_all_user(self): import cymysql #数据库所有用户信息 try: @@ -107,8 +99,7 @@ def pull_db_all_user(): conn.close() return rows - @staticmethod - def del_server_out_of_bound_safe(last_rows, rows): + def del_server_out_of_bound_safe(self, last_rows, rows): #停止超流量的服务 #启动没超流量的服务 #需要动态载入switchrule，以便实时修改规则 @@ -180,7 +171,7 @@ def del_server_out_of_bound_safe(last_rows, rows): if len(new_servers) > 0: from shadowsocks import eventloop - DbTransfer.get_instance().event.wait(eventloop.TIMEOUT_PRECISION) + self.event.wait(eventloop.TIMEOUT_PRECISION) for port in new_servers.keys(): passwd, cfg = new_servers[port] logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) @@ -196,32 +187,68 @@ def del_servers(): ServerPool.get_instance().cb_del_server(port) @staticmethod - def thread_db(): + def thread_db(obj): import socket import time + global db_instance timeout = 60 socket.setdefaulttimeout(timeout) last_rows = [] + db_instance = obj() try: while True: load_config() try: - DbTransfer.get_instance().push_db_all_user() - rows = DbTransfer.get_instance().pull_db_all_user() - DbTransfer.del_server_out_of_bound_safe(last_rows, rows) + db_instance.push_db_all_user() + rows = db_instance.pull_db_all_user() + db_instance.del_server_out_of_bound_safe(last_rows, rows) last_rows = rows except Exception as e: trace = traceback.format_exc() logging.error(trace) #logging.warn('db thread except:%s' % e) - if DbTransfer.get_instance().event.wait(get_config().MYSQL_UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive(): + if db_instance.event.wait(get_config().MYSQL_UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive(): break except KeyboardInterrupt as e: pass - DbTransfer.del_servers() + db_instance.del_servers() ServerPool.get_instance().stop() + db_instance = None @staticmethod def thread_db_stop(): - DbTransfer.get_instance().event.set() + global db_instance + db_instance.event.set() + +class MuJsonTransfer(DbTransfer): + def __init__(self): + super(MuJsonTransfer, self).__init__() + + def update_all_user(self, dt_transfer): + import json + rows = None + + config_path = "mudb.json" + with open(config_path, 'r+') as f: + rows = shell.parse_json_in_str(f.read().decode('utf8')) + for row in rows: + if "port" in row: + port = row["port"] + if port in dt_transfer: + row["u"] += dt_transfer[port][0] + row["d"] += dt_transfer[port][1] + + if rows: + output = json.dumps(rows, sort_keys=True, indent=4, separators=(',', ': ')) + with open(config_path, 'w') as f: + f.write(output) + + def pull_db_all_user(self): + rows = None + + config_path = "mudb.json" + with open(config_path, 'r+') as f: + rows = shell.parse_json_in_str(f.read().decode('utf8')) + + return rows diff --git a/mudb.json b/mudb.json new file mode 100644 index 00000000..22fad21c --- /dev/null +++ b/mudb.json @@ -0,0 +1,26 @@ +[ + { + "user": "admin", + "port": 443, + "u": 0, + "d": 0, + "transfer_enable": 1125899906842624, + "passwd": "admin", + "method": "aes-128-cfb", + "protocol": "auth_sha1_v2_compatible", + "obfs": "tls1.2_ticket_auth_compatible", + "enable": true + }, + { + "user": "user001", + "port": 10001, + "u": 0, + "d": 0, + "transfer_enable": 1099511627776, + "passwd": "abc123", + "method": "rc4-md5", + "protocol": "auth_sha1_v2_compatible", + "obfs": "http_post_compatible", + "enable": true + } +] diff --git a/server.py b/server.py index db375ad4..75ce3401 100644 --- a/server.py +++ b/server.py @@ -27,27 +27,28 @@ import server_pool import db_transfer from shadowsocks import shell - -#def test(): -# thread.start_new_thread(DbTransfer.thread_db, ()) -# Api.web_server() +from configloader import load_config, get_config class MainThread(threading.Thread): - def __init__(self): + def __init__(self, obj): threading.Thread.__init__(self) + self.obj = obj def run(self): - db_transfer.DbTransfer.thread_db() + self.obj.thread_db(self.obj) def stop(self): - db_transfer.DbTransfer.thread_db_stop() + self.obj.thread_db_stop() def main(): shell.check_python() if False: db_transfer.DbTransfer.thread_db() else: - thread = MainThread() + if get_config().API_INTERFACE == 'mudbjson': + thread = MainThread(db_transfer.MuJsonTransfer) + else: + thread = MainThread(db_transfer.DbTransfer) thread.start() try: while thread.is_alive(): diff --git a/switchrule.py b/switchrule.py index e102c634..5b2d3133 100644 --- a/switchrule.py +++ b/switchrule.py @@ -1,7 +1,6 @@ from configloader import load_config, get_config def getKeys(): - load_config() key_list = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] if get_config().API_INTERFACE == 'sspanelv3': key_list += ['method'] From d653c9bdb3ec9ab23876e929be386529b1be4929 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 15 Jun 2016 17:09:12 +0800 Subject: [PATCH 195/487] add forbidden_port --- shadowsocks/common.py | 25 +++++++++++++++++++++++++ shadowsocks/shell.py | 7 ++++++- shadowsocks/tcprelay.py | 33 ++++++++++++++++++++++++++++----- shadowsocks/udprelay.py | 8 ++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 68017d52..5a3c9b15 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -283,6 +283,31 @@ def __contains__(self, addr): else: return False +class PortRange(object): + def __init__(self, range_str): + self.range = set() + if type(range_str) == str: + range_str = range_str.split(',') + for item in range_str: + try: + int_range = item.split('-') + if len(int_range) == 1: + self.range.add(int(item)) + elif len(int_range) == 2: + int_range[0] = int(int_range[0]) + int_range[1] = int(int_range[1]) + if int_range[0] < 0: + int_range[0] = 0 + if int_range[1] > 65535: + int_range[1] = 65535 + i = int_range[0] + while i <= int_range[1]: + self.range.add(i) + except Exception as e: + logging.error(e) + + def __contains__(self, val): + return val in self.range def test_inet_conv(): ipv4 = b'8.8.4.4' diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index ed0cc42e..0d6d3386 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -23,7 +23,7 @@ import sys import getopt import logging -from shadowsocks.common import to_bytes, to_str, IPNetwork +from shadowsocks.common import to_bytes, to_str, IPNetwork, PortRange from shadowsocks import encrypt @@ -261,6 +261,11 @@ def get_config(is_local): except Exception as e: logging.error(e) sys.exit(2) + try: + config['forbidden_port'] = PortRange(config.get('forbidden_port', '')) + except Exception as e: + logging.error(e) + sys.exit(2) try: config['ignore_bind'] = \ IPNetwork(config.get('ignore_bind', '127.0.0.0/8,::1/128')) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 77331d89..7e3f91b0 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -160,6 +160,10 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._forbidden_iplist = config['forbidden_ip'] else: self._forbidden_iplist = None + if 'forbidden_port' in config: + self._forbidden_portset = config['forbidden_port'] + else: + self._forbidden_portset = None if is_local: self._chosen_server = self._get_a_server() fd_to_handlers[local_sock.fileno()] = self @@ -511,10 +515,15 @@ def _create_remote_socket(self, ip, port): if len(addrs) == 0: raise Exception("getaddrinfo failed for %s:%d" % (ip, port)) af, socktype, proto, canonname, sa = addrs[0] - if not self._remote_udp and self._forbidden_iplist: - if common.to_str(sa[0]) in self._forbidden_iplist: - raise Exception('IP %s is in forbidden list, reject' % - common.to_str(sa[0])) + if not self._remote_udp: + if self._forbidden_iplist: + if common.to_str(sa[0]) in self._forbidden_iplist: + raise Exception('IP %s is in forbidden list, reject' % + common.to_str(sa[0])) + if self._forbidden_portset: + if sa[1] in self._forbidden_portset: + raise Exception('Port %d is in forbidden list, reject' % + sa[1]) remote_sock = socket.socket(af, socktype, proto) self._remote_sock = remote_sock self._fd_to_handlers[remote_sock.fileno()] = self @@ -652,7 +661,21 @@ def _on_local_read(self): else: data = obfs_decode[0] try: - data = self._protocol.server_post_decrypt(data) + newdata = self._protocol.server_post_decrypt(data) + if data and not newdata: + data = self._protocol.server_pre_encrypt(data) + data = self._encryptor.encrypt(data) + data = self._obfs.server_encode(data) + try: + self._write_to_sock(data, self._local_sock) + except Exception as e: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) + self.destroy() + return + data = newdata except Exception as e: shell.print_exception(e) logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 3781d7db..5af77b08 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -301,6 +301,10 @@ def __init__(self, server, reqid_to_handlers, fd_to_handlers, loop, self._forbidden_iplist = config['forbidden_ip'] else: self._forbidden_iplist = None + if 'forbidden_port' in config: + self._forbidden_portset = config['forbidden_port'] + else: + self._forbidden_portset = None #fd_to_handlers[local_sock.fileno()] = self #local_sock.setblocking(False) #loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) @@ -1077,6 +1081,10 @@ def _handle_server(self): common.to_str(sa[0])) # drop return + if self._forbidden_portset: + if sa[1] in self._forbidden_portset: + raise Exception('Port %d is in forbidden list, reject' % + sa[1]) client = socket.socket(af, socktype, proto) client.setblocking(False) is_dns = False From 72aab7701f823c9dd1f204e8cc50647a8a7ddf32 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 15 Jun 2016 17:29:05 +0800 Subject: [PATCH 196/487] fix a tiny bug --- shadowsocks/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 5a3c9b15..d75602b7 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -303,6 +303,7 @@ def __init__(self, range_str): i = int_range[0] while i <= int_range[1]: self.range.add(i) + i += 1 except Exception as e: logging.error(e) From 8cbca008c6c428ccd13329aea5a33d22106251ad Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 15 Jun 2016 18:13:45 +0800 Subject: [PATCH 197/487] custom 'forbidden_port' for each user --- db_transfer.py | 20 +++++++++++++++++--- shadowsocks/common.py | 3 +++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 12e6e559..2e0864a6 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -119,10 +119,13 @@ def del_server_out_of_bound_safe(self, last_rows, rows): port = row['port'] passwd = common.to_bytes(row['passwd']) cfg = {'password': passwd} - for name in ['method', 'obfs', 'protocol']: + + read_config_keys = ['method', 'obfs', 'protocol', 'forbidden_port'] + for name in read_config_keys: if name in row and row[name]: cfg[name] = row[name] + merge_config_keys = ['password'] + read_config_keys for name in cfg.keys(): if hasattr(cfg[name], 'encode'): cfg[name] = cfg[name].encode('utf-8') @@ -141,13 +144,13 @@ def del_server_out_of_bound_safe(self, last_rows, rows): cfgchange = False if port in ServerPool.get_instance().tcp_servers_pool: relay = ServerPool.get_instance().tcp_servers_pool[port] - for name in ['password', 'method', 'obfs', 'protocol']: + for name in merge_config_keys: if name in cfg and cfg[name] != relay._config[name]: cfgchange = True break; if not cfgchange and port in ServerPool.get_instance().tcp_ipv6_servers_pool: relay = ServerPool.get_instance().tcp_ipv6_servers_pool[port] - for name in ['password', 'method', 'obfs', 'protocol']: + for name in merge_config_keys: if name in cfg and cfg[name] != relay._config[name]: cfgchange = True break; @@ -249,6 +252,17 @@ def pull_db_all_user(self): config_path = "mudb.json" with open(config_path, 'r+') as f: rows = shell.parse_json_in_str(f.read().decode('utf8')) + for row in rows: + try: + if 'forbidden_ip' in row: + row['forbidden_ip'] = common.IPNetwork(row['forbidden_ip']) + except Exception as e: + logging.error(e) + try: + if 'forbidden_port' in row: + row['forbidden_port'] = common.PortRange(row['forbidden_port']) + except Exception as e: + logging.error(e) return rows diff --git a/shadowsocks/common.py b/shadowsocks/common.py index d75602b7..ee6e6f2a 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -310,6 +310,9 @@ def __init__(self, range_str): def __contains__(self, val): return val in self.range + def __eq__(self, other): + return self.range == other.range + def test_inet_conv(): ipv4 = b'8.8.4.4' b = inet_pton(socket.AF_INET, ipv4) From 5c1b90b7745f56306674267f1de65661795c8e00 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 15 Jun 2016 18:34:05 +0800 Subject: [PATCH 198/487] custom 'forbidden_ip' for each user fix cmp for IPNetwork & PortRange --- db_transfer.py | 2 +- shadowsocks/common.py | 9 +++++++-- shadowsocks/shell.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 2e0864a6..91cb5637 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -120,7 +120,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): passwd = common.to_bytes(row['passwd']) cfg = {'password': passwd} - read_config_keys = ['method', 'obfs', 'protocol', 'forbidden_port'] + read_config_keys = ['method', 'obfs', 'protocol', 'forbidden_ip', 'forbidden_port'] for name in read_config_keys: if name in row and row[name]: cfg[name] = row[name] diff --git a/shadowsocks/common.py b/shadowsocks/common.py index ee6e6f2a..afb7bbe7 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -233,6 +233,7 @@ class IPNetwork(object): ADDRLENGTH = {socket.AF_INET: 32, socket.AF_INET6: 128, False: 0} def __init__(self, addrs): + self.addrs_str = addrs self._network_list_v4 = [] self._network_list_v6 = [] if type(addrs) == str: @@ -283,8 +284,12 @@ def __contains__(self, addr): else: return False + def __cmp__(self, other): + return cmp(self.addrs_str, other.addrs_str) + class PortRange(object): def __init__(self, range_str): + self.range_str = range_str self.range = set() if type(range_str) == str: range_str = range_str.split(',') @@ -310,8 +315,8 @@ def __init__(self, range_str): def __contains__(self, val): return val in self.range - def __eq__(self, other): - return self.range == other.range + def __cmp__(self, other): + return cmp(self.range_str, other.range_str) def test_inet_conv(): ipv4 = b'8.8.4.4' diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 0d6d3386..64fbef2a 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -199,7 +199,7 @@ def get_config(is_local): elif key == '--user': config['user'] = to_str(value) elif key == '--forbidden-ip': - config['forbidden_ip'] = to_str(value).split(',') + config['forbidden_ip'] = to_str(value) elif key in ('-h', '--help'): if is_local: print_local_help() From 2f8517190c1c71b7306267256ed0bd28b7deea58 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 16 Jun 2016 00:17:31 +0800 Subject: [PATCH 199/487] ignore empty --- shadowsocks/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index afb7bbe7..bfc099c1 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -297,7 +297,8 @@ def __init__(self, range_str): try: int_range = item.split('-') if len(int_range) == 1: - self.range.add(int(item)) + if item: + self.range.add(int(item)) elif len(int_range) == 2: int_range[0] = int(int_range[0]) int_range[1] = int(int_range[1]) From 8556fd3c1825fc41f9cf08ef9d4b816921d40530 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 16 Jun 2016 01:35:18 +0800 Subject: [PATCH 200/487] fix UDPRelay _forbidden_portset --- shadowsocks/udprelay.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 5af77b08..fff62c10 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -926,6 +926,10 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self._forbidden_iplist = config['forbidden_ip'] else: self._forbidden_iplist = None + if 'forbidden_port' in config: + self._forbidden_portset = config['forbidden_port'] + else: + self._forbidden_portset = None addrs = socket.getaddrinfo(self._listen_addr, self._listen_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) @@ -1083,8 +1087,10 @@ def _handle_server(self): return if self._forbidden_portset: if sa[1] in self._forbidden_portset: - raise Exception('Port %d is in forbidden list, reject' % + logging.debug('Port %d is in forbidden list, reject' % sa[1]) + # drop + return client = socket.socket(af, socktype, proto) client.setblocking(False) is_dns = False From 97785aae2e78258bf5c31431295edca0255a6c03 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 16 Jun 2016 22:42:23 +0800 Subject: [PATCH 201/487] fix recv data bug --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7e3f91b0..859e92cd 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -662,7 +662,7 @@ def _on_local_read(self): data = obfs_decode[0] try: newdata = self._protocol.server_post_decrypt(data) - if data and not newdata: + '''if data and not newdata: data = self._protocol.server_pre_encrypt(data) data = self._encryptor.encrypt(data) data = self._obfs.server_encode(data) @@ -674,7 +674,7 @@ def _on_local_read(self): traceback.print_exc() logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() - return + return''' data = newdata except Exception as e: shell.print_exception(e) From 9e64d1837c3210a6856c6e55370c5bcf23cb04af Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 20 Jun 2016 14:52:02 +0800 Subject: [PATCH 202/487] add mudbjson mgr --- apiconfig.py | 3 + db_transfer.py | 4 +- mujson_mgr.py | 218 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 mujson_mgr.py diff --git a/apiconfig.py b/apiconfig.py index 82415470..fba0a752 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -3,6 +3,9 @@ NODE_ID = 1 API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) +#mudb +MUDB_FILE = 'mudb.json' + # Mysql MYSQL_HOST = '127.0.0.1' MYSQL_PORT = 3306 diff --git a/db_transfer.py b/db_transfer.py index 91cb5637..7115fe26 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -231,7 +231,7 @@ def update_all_user(self, dt_transfer): import json rows = None - config_path = "mudb.json" + config_path = get_config().MUDB_FILE with open(config_path, 'r+') as f: rows = shell.parse_json_in_str(f.read().decode('utf8')) for row in rows: @@ -249,7 +249,7 @@ def update_all_user(self, dt_transfer): def pull_db_all_user(self): rows = None - config_path = "mudb.json" + config_path = get_config().MUDB_FILE with open(config_path, 'r+') as f: rows = shell.parse_json_in_str(f.read().decode('utf8')) for row in rows: diff --git a/mujson_mgr.py b/mujson_mgr.py new file mode 100644 index 00000000..7d8163ec --- /dev/null +++ b/mujson_mgr.py @@ -0,0 +1,218 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +import traceback +from shadowsocks import common, shell +from configloader import load_config, get_config +import getopt +import sys + +class MuJsonLoader(object): + def __init__(self): + self.json = None + + def load(self, path): + with open(path, 'r+') as f: + self.json = shell.parse_json_in_str(f.read().decode('utf8')) + + def save(self, path): + import json + if self.json: + output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) + with open(path, 'w') as f: + f.write(output) + +class MuMgr(object): + def __init__(self): + self.config_path = get_config().MUDB_FILE + self.data = MuJsonLoader() + + def userinfo(self, user): + ret = "" + for key in user.keys(): + ret += '\n' + if key in ['transfer_enable', 'u', 'd'] : + val = user[key] + if val / 1024 < 4: + ret += " %s : %s" % (key, val) + elif val / 1024**2 < 4: + val /= float(1024) + ret += " %s : %s K bytes" % (key, val) + elif val / 1024**3 < 4: + val /= float(1024**2) + ret += " %s : %s M bytes" % (key, val) + else: + val /= float(1024**3) + ret += " %s : %s G bytes" % (key, val) + else: + ret += " %s : %s" % (key, user[key]) + return ret + + def add(self, user): + up = {'enable': True, 'u': 0, 'd': 0, 'passwd': "m", 'method': "aes-128-cfb", + 'protocol': "auth_sha1_v2_compatible", + 'obfs': "tls1.2_ticket_auth_compatible", + 'transfer_enable': 1125899906842624} + up.update(user) + + self.data.load(self.config_path) + for row in self.data.json: + match = False + if 'user' in user and row['user'] == user['user']: + match = True + if 'port' in user and row['port'] == user['port']: + match = True + if match: + print("user [%s] port [%s] already exist" % (row['user'], row['port'])) + return + self.data.json.append(up) + print("### add user info %s" % self.userinfo(up)) + self.data.save(self.config_path) + + def edit(self, user): + self.data.load(self.config_path) + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("edit user [%s]" % (row['user'],)) + row.update(user) + print("### new user info %s" % self.userinfo(row)) + break + self.data.save(self.config_path) + + def delete(self, user): + self.data.load(self.config_path) + index = 0 + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("delete user [%s]" % row['user']) + del self.data.json[index] + break + index += 1 + self.data.save(self.config_path) + + def clear_ud(self, user): + up = {'u': 0, 'd': 0} + self.data.load(self.config_path) + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("clear user %s" % self.userinfo(row['user'])) + row.update(up) + self.data.save(self.config_path) + + def list_user(self, user): + self.data.load(self.config_path) + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("### user [%s] info %s" % (row['user'], self.userinfo(row))) + +def print_server_help(): + print('''usage: python mujson_manage.py -a|-d|-e|-c|-l [OPTION]... + +Actions: + -a ADD add/edit a user + -d DELETE delete a user + -e EDIT edit a user + -c CLEAR set u/d to zero + -l LIST display a user infomation or all users infomation + +Options: + -u USER the user name + -p PORT server port + -k PASSWORD password + -m METHOD encryption method, default: aes-128-cfb + -O PROTOCOL protocol plugin, default: auth_sha1_v2_compatible + -o OBFS obfs plugin, default: tls1.2_ticket_auth_compatible + -t TRANSFER max transfer for G bytes, default: 1048576, can be float point number + +General options: + -h, --help show this help message and exit +''') + +def main(): + shortopts = 'hadeclu:p:k:O:o:m:t' + longopts = ['help'] + action = None + user = {} + try: + optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + for key, value in optlist: + if key == '-a': + action = 1 + elif key == '-d': + action = 2 + elif key == '-e': + action = 3 + elif key == '-l': + action = 4 + elif key == '-c': + action = 0 + elif key == '-u': + user['user'] = value + elif key == '-p': + user['port'] = int(value) + elif key == '-k': + user['passwd'] = value + elif key == '-o': + user['obfs'] = value + elif key == '-O': + user['protocal'] = value + elif key == '-m': + user['method'] = value + elif key == '-t': + val = float(value) + try: + val = int(value) + except: + pass + user['transfer_enable'] = val * (1024 ** 3) + elif key in ('-h', '--help'): + print_server_help() + sys.exit(0) + except getopt.GetoptError as e: + print(e) + sys.exit(2) + + manage = MuMgr() + if action == 0: + manage.clear_ud(user) + elif action == 1: + if 'user' in user and 'port' in user: + manage.add(user) + else: + print("You have to set the user name and port with -u/-p") + elif action == 2: + if 'user' in user or 'port' in user: + manage.delete(user) + else: + print("You have to set the user name or port with -u/-p") + elif action == 3: + if 'user' in user or 'port' in user: + manage.edit(user) + else: + print("You have to set the user name or port with -u/-p") + elif action == 4: + manage.list_user(user) + +if __name__ == '__main__': + main() + From 268f6e23b44fb1e45ae10f2486c6a40332a036b3 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 20 Jun 2016 15:02:21 +0800 Subject: [PATCH 203/487] fix mujson_mgr & add -f --- mujson_mgr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 7d8163ec..94f57810 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -149,7 +149,7 @@ def print_server_help(): ''') def main(): - shortopts = 'hadeclu:p:k:O:o:m:t' + shortopts = 'adeclu:p:k:O:o:m:t:f:h' longopts = ['help'] action = None user = {} @@ -178,6 +178,8 @@ def main(): user['protocal'] = value elif key == '-m': user['method'] = value + elif key == '-f': + user['forbidden_port'] = value elif key == '-t': val = float(value) try: @@ -196,10 +198,12 @@ def main(): if action == 0: manage.clear_ud(user) elif action == 1: + if 'user' not in user and 'port' in user: + user['user'] = str(user['port']) if 'user' in user and 'port' in user: manage.add(user) else: - print("You have to set the user name and port with -u/-p") + print("You have to set the port with -p") elif action == 2: if 'user' in user or 'port' in user: manage.delete(user) From 044e1539e9f51613dd289713a1aa37ce05cbb0da Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 20 Jun 2016 17:19:04 +0800 Subject: [PATCH 204/487] fix clear command --- mujson_mgr.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 94f57810..c6403b94 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -110,12 +110,16 @@ def clear_ud(self, user): if 'port' in user and row['port'] != user['port']: match = False if match: - print("clear user %s" % self.userinfo(row['user'])) row.update(up) + print("clear user [%s]" % row['user']) self.data.save(self.config_path) def list_user(self, user): self.data.load(self.config_path) + if not user: + for row in self.data.json: + print("user [%s] port %s" % (row['user'], row['port'])) + return for row in self.data.json: match = True if 'user' in user and row['user'] != user['user']: From 8031337d653dc7de8516b428eac9f8cf9591adf0 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 21 Jun 2016 21:04:41 +0800 Subject: [PATCH 205/487] compatible python3 --- configloader.py | 22 +++++--------------- db_transfer.py | 32 +++++++++++++++++++----------- importloader.py | 22 ++++++++++++++++++++ mujson_mgr.py | 6 +++--- server_pool.py | 8 ++++---- shadowsocks/common.py | 7 +++---- shadowsocks/encrypt.py | 2 +- shadowsocks/obfsplugin/obfs_tls.py | 20 +++++++++---------- 8 files changed, 68 insertions(+), 51 deletions(-) create mode 100644 importloader.py diff --git a/configloader.py b/configloader.py index 8d0ec0d7..cf9d6196 100644 --- a/configloader.py +++ b/configloader.py @@ -1,27 +1,15 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- +import importloader -config = None +g_config = None def load_config(): - global config - try: - import userapiconfig - reload(userapiconfig) - config = userapiconfig - return - except: - pass - try: - import apiconfig - reload(apiconfig) - config = apiconfig - except: - pass + global g_config + g_config = importloader.loads(['userapiconfig', 'apiconfig']) def get_config(): - global config - return config + return g_config load_config() diff --git a/db_transfer.py b/db_transfer.py index 7115fe26..05bb31ad 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -8,7 +8,9 @@ import traceback from shadowsocks import common, shell from configloader import load_config, get_config +import importloader +switchrule = None db_instance = None class DbTransfer(object): @@ -80,8 +82,7 @@ def pull_db_all_user(self): import cymysql #数据库所有用户信息 try: - import switchrule - reload(switchrule) + switchrule = importloader.load('switchrule') keys = switchrule.getKeys() except Exception as e: keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] @@ -99,13 +100,19 @@ def pull_db_all_user(self): conn.close() return rows + def cmp(self, val1, val2): + if type(val1) is bytes: + val1 = common.to_str(val1) + if type(val2) is bytes: + val2 = common.to_str(val2) + return val1 == val2 + def del_server_out_of_bound_safe(self, last_rows, rows): #停止超流量的服务 #启动没超流量的服务 #需要动态载入switchrule，以便实时修改规则 try: - import switchrule - reload(switchrule) + switchrule = importloader.load('switchrule') except Exception as e: logging.error('load switchrule.py fail') cur_servers = {} @@ -145,13 +152,13 @@ def del_server_out_of_bound_safe(self, last_rows, rows): if port in ServerPool.get_instance().tcp_servers_pool: relay = ServerPool.get_instance().tcp_servers_pool[port] for name in merge_config_keys: - if name in cfg and cfg[name] != relay._config[name]: + if name in cfg and not self.cmp(cfg[name], relay._config[name]): cfgchange = True break; if not cfgchange and port in ServerPool.get_instance().tcp_ipv6_servers_pool: relay = ServerPool.get_instance().tcp_ipv6_servers_pool[port] for name in merge_config_keys: - if name in cfg and cfg[name] != relay._config[name]: + if name in cfg and not self.cmp(cfg[name], relay._config[name]): cfgchange = True break; #config changed @@ -182,10 +189,10 @@ def del_server_out_of_bound_safe(self, last_rows, rows): @staticmethod def del_servers(): - for port in ServerPool.get_instance().tcp_servers_pool.keys(): + for port in [v for v in ServerPool.get_instance().tcp_servers_pool.keys()]: if ServerPool.get_instance().server_is_run(port) > 0: ServerPool.get_instance().cb_del_server(port) - for port in ServerPool.get_instance().tcp_ipv6_servers_pool.keys(): + for port in [v for v in ServerPool.get_instance().tcp_ipv6_servers_pool.keys()]: if ServerPool.get_instance().server_is_run(port) > 0: ServerPool.get_instance().cb_del_server(port) @@ -232,8 +239,8 @@ def update_all_user(self, dt_transfer): rows = None config_path = get_config().MUDB_FILE - with open(config_path, 'r+') as f: - rows = shell.parse_json_in_str(f.read().decode('utf8')) + with open(config_path, 'rb+') as f: + rows = json.loads(f.read().decode('utf8')) for row in rows: if "port" in row: port = row["port"] @@ -247,11 +254,12 @@ def update_all_user(self, dt_transfer): f.write(output) def pull_db_all_user(self): + import json rows = None config_path = get_config().MUDB_FILE - with open(config_path, 'r+') as f: - rows = shell.parse_json_in_str(f.read().decode('utf8')) + with open(config_path, 'rb+') as f: + rows = json.loads(f.read().decode('utf8')) for row in rows: try: if 'forbidden_ip' in row: diff --git a/importloader.py b/importloader.py new file mode 100644 index 00000000..fb7c26b7 --- /dev/null +++ b/importloader.py @@ -0,0 +1,22 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +def load(name): + try: + obj = __import__(name) + return obj + except: + pass + + try: + import importlib + obj = importlib.__import__(name) + return obj + except: + pass + +def loads(namelist): + for name in namelist: + obj = load(name) + if obj is not None: + return obj diff --git a/mujson_mgr.py b/mujson_mgr.py index c6403b94..cda7ea77 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -6,17 +6,17 @@ from configloader import load_config, get_config import getopt import sys +import json class MuJsonLoader(object): def __init__(self): self.json = None def load(self, path): - with open(path, 'r+') as f: - self.json = shell.parse_json_in_str(f.read().decode('utf8')) + with open(path, 'rb+') as f: + self.json = json.loads(f.read().decode('utf8')) def save(self, path): - import json if self.json: output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) with open(path, 'w') as f: diff --git a/server_pool.py b/server_pool.py index 63a8f16f..c25c832d 100644 --- a/server_pool.py +++ b/server_pool.py @@ -24,7 +24,7 @@ import os import logging import time -from shadowsocks import shell, eventloop, tcprelay, udprelay, asyncdns +from shadowsocks import shell, eventloop, tcprelay, udprelay, asyncdns, common import threading import sys from socket import * @@ -124,7 +124,7 @@ def new_server(self, port, user_config): a_config['server_port'] = port a_config['max_connect'] = 128 try: - logging.info("starting server at [%s]:%d" % (a_config['server'], port)) + logging.info("starting server at [%s]:%d" % (common.to_str(a_config['server']), port)) tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False, stat_counter=self.stat_counter) tcp_server.add_to_loop(self.loop) @@ -134,14 +134,14 @@ def new_server(self, port, user_config): udp_server.add_to_loop(self.loop) self.udp_ipv6_servers_pool.update({port: udp_server}) - if a_config['server_ipv6'] == "::": + if common.to_str(a_config['server_ipv6']) == "::": ipv6_ok = True except Exception as e: logging.warn("IPV6 %s " % (e,)) if 'server' in self.config: if port in self.tcp_servers_pool: - logging.info("server already at %s:%d" % (self.config['server'], port)) + logging.info("server already at %s:%d" % (common.to_str(self.config['server']), port)) return 'this port server is already running' else: a_config = self.config.copy() diff --git a/shadowsocks/common.py b/shadowsocks/common.py index bfc099c1..16386bda 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -85,7 +85,7 @@ def inet_pton(family, addr): if '.' in addr: # a v4 addr v4addr = addr[addr.rindex(':') + 1:] v4addr = socket.inet_aton(v4addr) - v4addr = map(lambda x: ('%02X' % ord(x)), v4addr) + v4addr = ['%02X' % ord(x) for x in v4addr] v4addr.insert(2, ':') newaddr = addr[:addr.rindex(':') + 1] + ''.join(v4addr) return inet_pton(family, newaddr) @@ -289,10 +289,9 @@ def __cmp__(self, other): class PortRange(object): def __init__(self, range_str): - self.range_str = range_str + self.range_str = to_str(range_str) self.range = set() - if type(range_str) == str: - range_str = range_str.split(',') + range_str = to_str(range_str).split(',') for item in range_str: try: int_range = item.split('-') diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index e8f51b74..57540bca 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -75,7 +75,7 @@ def EVP_BytesToKey(password, key_len, iv_len): class Encryptor(object): def __init__(self, key, method): self.key = key - self.method = method + self.method = common.to_str(method) self.iv = None self.iv_sent = False self.cipher_iv = b'' diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 00ca0454..f483d46d 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -169,8 +169,8 @@ def client_encode(self, buf): data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data return data if self.has_recv_header: - data = b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + "\x00\x20" + os.urandom(22) #Finished + data = b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished data += hmac.new(self.server_info.key + self.server_info.data.client_id, data, hashlib.sha1).digest()[:10] ret = data + self.send_buffer self.send_buffer = b'' @@ -196,8 +196,8 @@ def server_encode(self, buf): data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"0016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data - data += b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + "\x00\x20" + os.urandom(22) #Finished + data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] return data @@ -326,8 +326,8 @@ def client_encode(self, buf): data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data return data elif self.handshake_status == 1 and len(buf) == 0: - data = b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + "\x00\x20" + os.urandom(22) #Finished + data = b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished data += hmac.new(self.server_info.key + self.server_info.data.client_id, data, hashlib.sha1).digest()[:10] ret = data + self.send_buffer self.send_buffer = b'' @@ -376,8 +376,8 @@ def server_encode(self, buf): data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"c02f000005ff01000100") data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data - data += b"\x14" + self.tls_version + "\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + "\x00\x20" + os.urandom(22) #Finished + data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec + data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] return data @@ -410,10 +410,10 @@ def server_decode(self, buf): verify_len = 43 - 10 if len(buf) < 43: raise Exception('server_decode data error') - if not match_begin(buf, b"\x14" + self.tls_version + "\x00\x01\x01"): #ChangeCipherSpec + if not match_begin(buf, b"\x14" + self.tls_version + b"\x00\x01\x01"): #ChangeCipherSpec raise Exception('server_decode data error') buf = buf[6:] - if not match_begin(buf, b"\x16" + self.tls_version + "\x00\x20"): #Finished + if not match_begin(buf, b"\x16" + self.tls_version + b"\x00\x20"): #Finished raise Exception('server_decode data error') if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: raise Exception('server_decode data error') From 44598a070d23e3ddba6d92d805d1419ab17c0d60 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 21 Jun 2016 21:29:34 +0800 Subject: [PATCH 206/487] compatible python3 --- shadowsocks/encrypt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 57540bca..9609f89a 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -75,7 +75,8 @@ def EVP_BytesToKey(password, key_len, iv_len): class Encryptor(object): def __init__(self, key, method): self.key = key - self.method = common.to_str(method) + method = common.to_str(method) + self.method = method self.iv = None self.iv_sent = False self.cipher_iv = b'' From acb3eb2910425d12f2a716d5ec5079bd14835c01 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 21 Jun 2016 22:42:55 +0800 Subject: [PATCH 207/487] compatible python3 --- server_pool.py | 4 +++- shadowsocks/encrypt.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server_pool.py b/server_pool.py index c25c832d..ee0cad26 100644 --- a/server_pool.py +++ b/server_pool.py @@ -123,6 +123,7 @@ def new_server(self, port, user_config): a_config['server'] = a_config['server_ipv6'] a_config['server_port'] = port a_config['max_connect'] = 128 + a_config['method'] = common.to_str(a_config['method']) try: logging.info("starting server at [%s]:%d" % (common.to_str(a_config['server']), port)) @@ -148,8 +149,9 @@ def new_server(self, port, user_config): a_config.update(user_config) a_config['server_port'] = port a_config['max_connect'] = 128 + a_config['method'] = common.to_str(a_config['method']) try: - logging.info("starting server at %s:%d" % (a_config['server'], port)) + logging.info("starting server at %s:%d" % (common.to_str(a_config['server']), port)) tcp_server = tcprelay.TCPRelay(a_config, self.dns_resolver, False) tcp_server.add_to_loop(self.loop) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 9609f89a..e8f51b74 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -75,7 +75,6 @@ def EVP_BytesToKey(password, key_len, iv_len): class Encryptor(object): def __init__(self, key, method): self.key = key - method = common.to_str(method) self.method = method self.iv = None self.iv_sent = False From 3d56b040cffbd98899c95963379d232476d05530 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 21 Jun 2016 23:26:00 +0800 Subject: [PATCH 208/487] accept older connection id --- shadowsocks/obfsplugin/auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index d64c98b5..0f2ee35a 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -103,8 +103,9 @@ def is_active(self): def re_enable(self, connection_id): self.enable = True self.alloc = {} - self.front = connection_id + self.front = connection_id - 16 self.back = connection_id + self.alloc[connection_id] = 1 def insert(self, connection_id): self.update() From 3d1d38aa0f7d16ac8f1a2b0ccebb66430c042091 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 01:14:34 +0800 Subject: [PATCH 209/487] fix connections stat mgr help info --- mujson_mgr.py | 1 + shadowsocks/tcprelay.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index cda7ea77..d986ca5b 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -147,6 +147,7 @@ def print_server_help(): -O PROTOCOL protocol plugin, default: auth_sha1_v2_compatible -o OBFS obfs plugin, default: tls1.2_ticket_auth_compatible -t TRANSFER max transfer for G bytes, default: 1048576, can be float point number + -f FORBID set forbidden ports. Example (ban 1~79 and 81~100): -f "1-79,81-100" General options: -h, --help show this help message and exit diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 859e92cd..1349f62f 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -994,14 +994,11 @@ def stat_add(self, local_addr, val): self._stat_counter[self._listen_port] = {} newval = self._stat_counter[self._listen_port].get(local_addr, 0) + val logging.debug('port %d addr %s connections %d' % (self._listen_port, local_addr, newval)) + self._stat_counter[self._listen_port][local_addr] = newval + self.update_stat(self._listen_port, self._stat_counter[self._listen_port], val) if newval <= 0: if local_addr in self._stat_counter[self._listen_port]: del self._stat_counter[self._listen_port][local_addr] - if len(self._stat_counter[self._listen_port]) == 0: - del self._stat_counter[self._listen_port] - else: - self._stat_counter[self._listen_port][local_addr] = newval - self.update_stat(self._listen_port, self._stat_counter[self._listen_port], val) newval = self._stat_counter.get(0, 0) + val self._stat_counter[0] = newval From d2b6c34bdf1799494fe764017e934ee592cca089 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 01:32:35 +0800 Subject: [PATCH 210/487] accept older connection id --- shadowsocks/obfsplugin/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 0f2ee35a..482b13a0 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -103,7 +103,7 @@ def is_active(self): def re_enable(self, connection_id): self.enable = True self.alloc = {} - self.front = connection_id - 16 + self.front = connection_id - 64 self.back = connection_id self.alloc[connection_id] = 1 From 3912ecba8e843a62802279ca1f5e26a85b675f06 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 01:36:36 +0800 Subject: [PATCH 211/487] accept older connection id --- shadowsocks/obfsplugin/auth.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 482b13a0..a56645c4 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -88,9 +88,9 @@ def server_decode(self, buf): class client_queue(object): def __init__(self, begin_id): - self.front = begin_id - self.back = begin_id - self.alloc = {} + self.front = begin_id - 64 + self.back = begin_id + 1 + self.alloc = {begin_id: 1} self.enable = True self.last_update = time.time() @@ -102,10 +102,9 @@ def is_active(self): def re_enable(self, connection_id): self.enable = True - self.alloc = {} self.front = connection_id - 64 - self.back = connection_id - self.alloc[connection_id] = 1 + self.back = connection_id + 1 + self.alloc = {connection_id: 1} def insert(self, connection_id): self.update() From 7259b211c654f12f309eaf92054ccd534d4f1304 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 02:17:31 +0800 Subject: [PATCH 212/487] fix auth, refine log --- shadowsocks/obfsplugin/auth.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index a56645c4..2197a5b9 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -90,7 +90,7 @@ class client_queue(object): def __init__(self, begin_id): self.front = begin_id - 64 self.back = begin_id + 1 - self.alloc = {begin_id: 1} + self.alloc = {} self.enable = True self.last_update = time.time() @@ -104,23 +104,23 @@ def re_enable(self, connection_id): self.enable = True self.front = connection_id - 64 self.back = connection_id + 1 - self.alloc = {connection_id: 1} + self.alloc = {} def insert(self, connection_id): - self.update() if not self.enable: - logging.warn('auth_simple: not enable') - return False - if connection_id < self.front: - logging.warn('auth_simple: duplicate id') + logging.warn('obfs auth: not enable') return False if not self.is_active(): self.re_enable(connection_id) + self.update() + if connection_id < self.front: + logging.warn('obfs auth: duplicate id') + return False if connection_id > self.front + 0x4000: - logging.warn('auth_simple: wrong id') + logging.warn('obfs auth: wrong id') return False if connection_id in self.alloc: - logging.warn('auth_simple: duplicate id 2') + logging.warn('obfs auth: duplicate id 2') return False if self.back <= connection_id: self.back = connection_id + 1 @@ -154,7 +154,7 @@ def insert(self, client_id, connection_id): if self.client_id[c_id].is_active(): active += 1 if active >= self.max_client: - logging.warn('auth_simple: max active clients exceeded') + logging.warn('obfs auth: max active clients exceeded') return False if len(self.client_id) < self.max_client: @@ -176,7 +176,7 @@ def insert(self, client_id, connection_id): else: self.client_id[client_id].re_enable(connection_id) return self.client_id[client_id].insert(connection_id) - logging.warn('auth_simple: no inactive client [assert]') + logging.warn('obfs auth: no inactive client [assert]') return False else: return self.client_id[client_id].insert(connection_id) From ad625cd89964aa139a69cb91c089549dc566e89d Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 16:07:57 +0800 Subject: [PATCH 213/487] custom dns ip & port with 'dns.conf' --- apiconfig.py | 1 + shadowsocks/asyncdns.py | 45 ++++++++++++++++++++++++++++++----------- shadowsocks/obfs.py | 2 +- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/apiconfig.py b/apiconfig.py index fba0a752..bab59534 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,6 +1,7 @@ # Config TRANSFER_MUL = 1.0 NODE_ID = 1 +SERVER_PUB_ADDR = '127.0.0.1' API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) #mudb diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index ebd5e9e9..3273c360 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -284,23 +284,44 @@ def __init__(self): def _parse_resolv(self): self._servers = [] try: - with open('/etc/resolv.conf', 'rb') as f: + with open('dns.conf', 'rb') as f: content = f.readlines() for line in content: line = line.strip() if line: - if line.startswith(b'nameserver'): - parts = line.split() - if len(parts) >= 2: - server = parts[1] - if common.is_ip(server) == socket.AF_INET: - if type(server) != str: - server = server.decode('utf8') - self._servers.append(server) + parts = line.split(' ', 1) + if len(parts) >= 2: + server = parts[0] + port = int(parts[1]) + else: + server = parts[0] + port = 53 + if common.is_ip(server) == socket.AF_INET: + if type(server) != str: + server = server.decode('utf8') + self._servers.append((server, port)) except IOError: pass if not self._servers: - self._servers = ['8.8.4.4', '8.8.8.8'] + try: + with open('/etc/resolv.conf', 'rb') as f: + content = f.readlines() + for line in content: + line = line.strip() + if line: + if line.startswith(b'nameserver'): + parts = line.split() + if len(parts) >= 2: + server = parts[1] + if common.is_ip(server) == socket.AF_INET: + if type(server) != str: + server = server.decode('utf8') + self._servers.append((server, 53)) + except IOError: + pass + if not self._servers: + self._servers = [('8.8.4.4', 53), ('8.8.8.8', 53)] + logging.info('dns server: %s' % (self._servers,)) def _parse_hosts(self): etc_path = '/etc/hosts' @@ -400,7 +421,7 @@ def handle_event(self, sock, fd, event): self._loop.add(self._sock, eventloop.POLL_IN, self) else: data, addr = sock.recvfrom(1024) - if addr[0] not in self._servers: + if addr not in self._servers: logging.warn('received a packet other than our dns') return self._handle_data(data) @@ -425,7 +446,7 @@ def _send_req(self, hostname, qtype): for server in self._servers: logging.debug('resolving %s with type %d using server %s', hostname, qtype, server) - self._sock.sendto(req, (server, 53)) + self._sock.sendto(req, server) def resolve(self, hostname, callback): if type(hostname) != bytes: diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index f34c7c0f..68a92f14 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -45,7 +45,7 @@ def __init__(self, method): if self._method_info: self.obfs = self.get_obfs(method) else: - raise Exception('method %s not supported' % method) + raise Exception('obfs plugin [%s] not supported' % method) def init_data(self): return self.obfs.init_data() From 9b14c019bb311b9b9ab86ff2178a511025c405e2 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 17:11:16 +0800 Subject: [PATCH 214/487] obfs http_simple broken in some cases --- shadowsocks/obfsplugin/http_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 10cde0f7..00568670 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -172,7 +172,7 @@ def server_decode(self, buf): ret_buf = self.get_data_from_http_header(buf) if len(datas) > 1: ret_buf += datas[1] - if len(ret_buf) >= 15: + if len(ret_buf) >= 7: self.has_recv_header = True return (ret_buf, True, False) if len(ret_buf) == 0: @@ -241,7 +241,7 @@ def server_decode(self, buf): ret_buf = self.get_data_from_http_header(buf) if len(datas) > 1: ret_buf += datas[1] - if len(ret_buf) >= 15: + if len(ret_buf) >= 7: self.has_recv_header = True return (ret_buf, True, False) if len(ret_buf) == 0: From b344bc236b5a28ada617e462b3a4f8dcc8fb595f Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 17:37:51 +0800 Subject: [PATCH 215/487] fix bytes --- shadowsocks/asyncdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 3273c360..3f92e628 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -289,7 +289,7 @@ def _parse_resolv(self): for line in content: line = line.strip() if line: - parts = line.split(' ', 1) + parts = line.split(b' ', 1) if len(parts) >= 2: server = parts[0] port = int(parts[1]) From 4b2adda44072887bd449d9b53778744caec327f6 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 22 Jun 2016 17:46:08 +0800 Subject: [PATCH 216/487] print_server_help by default --- mujson_mgr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mujson_mgr.py b/mujson_mgr.py index d986ca5b..0d5c13dd 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -221,6 +221,8 @@ def main(): print("You have to set the user name or port with -u/-p") elif action == 4: manage.list_user(user) + elif action is None: + print_server_help() if __name__ == '__main__': main() From f58d76da478e27aa8815b77585af4ee1657c6a28 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 23 Jun 2016 13:58:30 +0800 Subject: [PATCH 217/487] fix spell --- mujson_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 0d5c13dd..092baf85 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -180,7 +180,7 @@ def main(): elif key == '-o': user['obfs'] = value elif key == '-O': - user['protocal'] = value + user['protocol'] = value elif key == '-m': user['method'] = value elif key == '-f': From bd43069bcfd43e0c4793ece5a8b3482295bd4691 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 24 Jun 2016 19:02:31 +0800 Subject: [PATCH 218/487] sendback param of server_post_decrypt details of errors --- shadowsocks/obfsplugin/auth.py | 63 +++++++++++++++++--------------- shadowsocks/obfsplugin/plain.py | 2 +- shadowsocks/obfsplugin/verify.py | 26 ++++++------- shadowsocks/tcprelay.py | 26 +++++++------ 4 files changed, 62 insertions(+), 55 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 2197a5b9..012236ff 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -277,7 +277,7 @@ def server_pre_encrypt(self, buf): def server_post_decrypt(self, buf): if self.raw_trans: - return buf + return (buf, False) self.recv_buf += buf out_buf = b'' while len(self.recv_buf) > 2: @@ -287,7 +287,7 @@ def server_post_decrypt(self, buf): self.recv_buf = b'' if self.decrypt_packet_num == 0: logging.info('auth_simple: over size') - return b'E' + return (b'E', False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -298,7 +298,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return b'E' + return (b'E', False) else: raise Exception('server_post_decrype data uncorrect CRC32') @@ -309,7 +309,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' logging.info('auth_simple: too short') - return b'E' + return (b'E', False) utc_time = struct.unpack('H', self.recv_buf[4:6])[0] if length > len(self.recv_buf): - return b'' + return (b'', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] if sha1data != self.recv_buf[length - 10:length]: logging.error('auth_sha1 data uncorrect auth HMAC-SHA1') - return b'E' + return (b'E', False) pos = common.ord(self.recv_buf[6]) + 6 out_buf = self.recv_buf[pos:length - 10] if len(out_buf) < 12: self.raw_trans = True self.recv_buf = b'' logging.info('auth_sha1: too short') - return b'E' + return (b'E', False) utc_time = struct.unpack(' len(self.recv_buf): @@ -514,7 +514,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return b'E' + return (b'E', False) else: raise Exception('server_post_decrype data uncorrect checksum') @@ -525,7 +525,7 @@ def server_post_decrypt(self, buf): if out_buf: self.server_info.data.update(self.client_id, self.connection_id) self.decrypt_packet_num += 1 - return out_buf + return (out_buf, False) class obfs_auth_v2_data(object): def __init__(self): @@ -686,26 +686,26 @@ def server_pre_encrypt(self, buf): def server_post_decrypt(self, buf): if self.raw_trans: - return buf + return (buf, False) self.recv_buf += buf out_buf = b'' if not self.has_recv_header: if len(self.recv_buf) < 4: - return b'' + return (b'', False) crc = struct.pack('H', self.recv_buf[4:6])[0] if length > len(self.recv_buf): - return b'' + return (b'', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] if sha1data != self.recv_buf[length - 10:length]: logging.error('auth_sha1_v2 data uncorrect auth HMAC-SHA1') - return b'E' + return (b'E', False) pos = common.ord(self.recv_buf[6]) if pos < 255: pos += 6 @@ -716,7 +716,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' logging.info('auth_sha1_v2: too short') - return b'E' + return (b'E', False) client_id = struct.unpack(' 2: length = struct.unpack('>H', self.recv_buf[:2])[0] if length >= 8192 or length < 7: @@ -739,7 +740,7 @@ def server_post_decrypt(self, buf): self.recv_buf = b'' if self.decrypt_packet_num == 0: logging.info('auth_sha1_v2: over size') - return b'E' + return (b'E', False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -750,7 +751,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return b'E' + return (b'E', False) else: raise Exception('server_post_decrype data uncorrect checksum') @@ -761,9 +762,11 @@ def server_post_decrypt(self, buf): pos = struct.unpack('>H', self.recv_buf[3:5])[0] + 2 out_buf += self.recv_buf[pos:length - 4] self.recv_buf = self.recv_buf[length:] + if pos == length - 4: + sendback = True if out_buf: self.server_info.data.update(self.client_id, self.connection_id) self.decrypt_packet_num += 1 - return out_buf + return (out_buf, sendback) diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index 8bb0000b..f8a733ad 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -67,7 +67,7 @@ def server_decode(self, buf): return (buf, True, False) def server_post_decrypt(self, buf): - return buf + return (buf, False) def client_udp_pre_encrypt(self, buf): return buf diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 639ba5cb..8f6b8c6a 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -151,7 +151,7 @@ def server_pre_encrypt(self, buf): def server_post_decrypt(self, buf): if self.raw_trans: - return buf + return (buf, False) self.recv_buf += buf out_buf = b'' while len(self.recv_buf) > 2: @@ -160,7 +160,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return b'E' + return (b'E', False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -170,7 +170,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return b'E' + return (b'E', False) else: raise Exception('server_post_decrype data uncorrect CRC32') @@ -180,7 +180,7 @@ def server_post_decrypt(self, buf): if out_buf: self.decrypt_packet_num += 1 - return out_buf + return (out_buf, False) class verify_deflate(verify_base): def __init__(self, method): @@ -236,7 +236,7 @@ def server_pre_encrypt(self, buf): def server_post_decrypt(self, buf): if self.raw_trans: - return buf + return (buf, False) self.recv_buf += buf out_buf = b'' while len(self.recv_buf) > 2: @@ -256,7 +256,7 @@ def server_post_decrypt(self, buf): if out_buf: self.decrypt_packet_num += 1 - return out_buf + return (out_buf, False) class verify_sha1(verify_base): def __init__(self, method): @@ -303,26 +303,26 @@ def server_pre_encrypt(self, buf): def server_post_decrypt(self, buf): if self.raw_trans: - return buf + return (buf, False) self.recv_buf += buf out_buf = b'' if not self.has_recv_header: if len(self.recv_buf) < 2: - return b'' + return (b'', False) if (ord(self.recv_buf[0]) & 0x10) != 0x10: if self.method == 'verify_sha1': logging.error('Not One-time authentication header') - return b'E' + return (b'E', False) else: self.raw_trans = True - return self.recv_buf + return (self.recv_buf, False) head_size = self.get_head_size(self.recv_buf, 30) if len(self.recv_buf) < head_size + 10: - return b'' + return (b'', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:head_size], hashlib.sha1).digest()[:10] if sha1data != self.recv_buf[head_size:head_size + 10]: logging.error('server_post_decrype data uncorrect auth HMAC-SHA1') - return b'E' + return (b'E', False) out_buf = to_bytes(chr(ord(self.recv_buf[0]) & 0xEF)) + self.recv_buf[1:head_size] self.recv_buf = self.recv_buf[head_size + 10:] self.has_recv_header = True @@ -340,7 +340,7 @@ def server_post_decrypt(self, buf): out_buf += data self.recv_buf = self.recv_buf[length:] - return out_buf + return (out_buf, False) def client_udp_pre_encrypt(self, buf): ret = self.pack_auth_data(buf) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 1349f62f..121f056b 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -364,7 +364,7 @@ def _get_redirect_host(self, client_address, ogn_data): return (host_post, 80) def _handel_protocol_error(self, client_address, ogn_data): - logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1])) + logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d via port %d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1], self._server._listen_port)) self._encrypt_correct = False #create redirect or disconnect by hash code host, port = self._get_redirect_host(client_address, ogn_data) @@ -518,12 +518,17 @@ def _create_remote_socket(self, ip, port): if not self._remote_udp: if self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: + if self._remote_address: + raise Exception('IP %s is in forbidden list, when connect to %s:%d via port %d' % + (common.to_str(sa[0]), self._remote_address[0], self._remote_address[1], self._server._listen_port)) raise Exception('IP %s is in forbidden list, reject' % common.to_str(sa[0])) if self._forbidden_portset: if sa[1] in self._forbidden_portset: - raise Exception('Port %d is in forbidden list, reject' % - sa[1]) + if self._remote_address: + raise Exception('Port %d is in forbidden list, when connect to %s:%d via port %d' % + (sa[1], self._remote_address[0], self._remote_address[1], self._server._listen_port)) + raise Exception('Port %d is in forbidden list, reject' % sa[1]) remote_sock = socket.socket(af, socktype, proto) self._remote_sock = remote_sock self._fd_to_handlers[remote_sock.fileno()] = self @@ -661,21 +666,20 @@ def _on_local_read(self): else: data = obfs_decode[0] try: - newdata = self._protocol.server_post_decrypt(data) - '''if data and not newdata: - data = self._protocol.server_pre_encrypt(data) - data = self._encryptor.encrypt(data) - data = self._obfs.server_encode(data) + data, sendback = self._protocol.server_post_decrypt(data) + if sendback: + backdata = self._protocol.server_pre_encrypt('') + backdata = self._encryptor.encrypt(backdata) + backdata = self._obfs.server_encode(backdata) try: - self._write_to_sock(data, self._local_sock) + self._write_to_sock(backdata, self._local_sock) except Exception as e: shell.print_exception(e) if self._config['verbose']: traceback.print_exc() logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() - return''' - data = newdata + return except Exception as e: shell.print_exception(e) logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) From 187f0de25af5f511eed76c7623195eb0259a8d57 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 24 Jun 2016 19:37:25 +0800 Subject: [PATCH 219/487] generate random password remove default users in mudb.json --- mudb.json | 24 ------------------------ mujson_mgr.py | 7 ++++++- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/mudb.json b/mudb.json index 22fad21c..0d4f101c 100644 --- a/mudb.json +++ b/mudb.json @@ -1,26 +1,2 @@ [ - { - "user": "admin", - "port": 443, - "u": 0, - "d": 0, - "transfer_enable": 1125899906842624, - "passwd": "admin", - "method": "aes-128-cfb", - "protocol": "auth_sha1_v2_compatible", - "obfs": "tls1.2_ticket_auth_compatible", - "enable": true - }, - { - "user": "user001", - "port": 10001, - "u": 0, - "d": 0, - "transfer_enable": 1099511627776, - "passwd": "abc123", - "method": "rc4-md5", - "protocol": "auth_sha1_v2_compatible", - "obfs": "http_post_compatible", - "enable": true - } ] diff --git a/mujson_mgr.py b/mujson_mgr.py index 092baf85..4518c6f4 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -4,6 +4,7 @@ import traceback from shadowsocks import common, shell from configloader import load_config, get_config +import random import getopt import sys import json @@ -48,11 +49,15 @@ def userinfo(self, user): ret += " %s : %s" % (key, user[key]) return ret + def rand_pass(self): + return b''.join([random.choice(b'''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i in range(8)]) + def add(self, user): - up = {'enable': True, 'u': 0, 'd': 0, 'passwd': "m", 'method': "aes-128-cfb", + up = {'enable': True, 'u': 0, 'd': 0, 'method': "aes-128-cfb", 'protocol': "auth_sha1_v2_compatible", 'obfs': "tls1.2_ticket_auth_compatible", 'transfer_enable': 1125899906842624} + up['passwd'] = self.rand_pass() up.update(user) self.data.load(self.config_path) From f5434bdbd02290050140f3b396bea6bb773f2850 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 25 Jun 2016 00:17:41 +0800 Subject: [PATCH 220/487] process empty str --- shadowsocks/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 16386bda..e1f574c8 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -150,6 +150,8 @@ def pack_addr(address): return b'\x03' + chr(len(address)) + address def pre_parse_header(data): + if not data: + return None datatype = ord(data[0]) if datatype == 0x80: if len(data) <= 2: From 050270a7258a3478797eb92bc647590f52b190a7 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 25 Jun 2016 00:39:04 +0800 Subject: [PATCH 221/487] fix send back str type --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 121f056b..e1898fc7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -668,7 +668,7 @@ def _on_local_read(self): try: data, sendback = self._protocol.server_post_decrypt(data) if sendback: - backdata = self._protocol.server_pre_encrypt('') + backdata = self._protocol.server_pre_encrypt(b'') backdata = self._encryptor.encrypt(backdata) backdata = self._obfs.server_encode(backdata) try: From 2621e5e5642c896cd8ac6545c1a8c3a7ca978f3c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 25 Jun 2016 14:01:47 +0800 Subject: [PATCH 222/487] fix mgr under py3 --- mujson_mgr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 4518c6f4..e3e4d834 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -2,7 +2,7 @@ # -*- coding: UTF-8 -*- import traceback -from shadowsocks import common, shell +from shadowsocks import shell from configloader import load_config, get_config import random import getopt @@ -50,7 +50,7 @@ def userinfo(self, user): return ret def rand_pass(self): - return b''.join([random.choice(b'''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i in range(8)]) + return ''.join([random.choice('''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i in range(8)]) def add(self, user): up = {'enable': True, 'u': 0, 'd': 0, 'method': "aes-128-cfb", From 50b11fff028222d4c1de73378286db49978e1fe5 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 25 Jun 2016 14:14:55 +0800 Subject: [PATCH 223/487] wait longer to keep safe --- db_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index 05bb31ad..0da6fe4d 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -181,7 +181,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): if len(new_servers) > 0: from shadowsocks import eventloop - self.event.wait(eventloop.TIMEOUT_PRECISION) + self.event.wait(eventloop.TIMEOUT_PRECISION + eventloop.TIMEOUT_PRECISION / 2) for port in new_servers.keys(): passwd, cfg = new_servers[port] logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) From 45584dc4f5bea7cdc0c8e6e079d08fdeea41d115 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 26 Jun 2016 21:23:39 +0800 Subject: [PATCH 224/487] display ssr link --- apiconfig.py | 2 +- mujson_mgr.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apiconfig.py b/apiconfig.py index bab59534..b69c50eb 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,7 +1,7 @@ # Config TRANSFER_MUL = 1.0 NODE_ID = 1 -SERVER_PUB_ADDR = '127.0.0.1' +SERVER_PUB_ADDR = '127.0.0.1' # mujson_mgr need this to generate ssr link API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) #mudb diff --git a/mujson_mgr.py b/mujson_mgr.py index e3e4d834..ff99ac23 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -8,6 +8,7 @@ import getopt import sys import json +import base64 class MuJsonLoader(object): def __init__(self): @@ -26,8 +27,17 @@ def save(self, path): class MuMgr(object): def __init__(self): self.config_path = get_config().MUDB_FILE + self.server_addr = get_config().SERVER_PUB_ADDR self.data = MuJsonLoader() + def ssrlink(self, user): + protocol = user.get('protocol', '') + obfs = user.get('obfs', '') + protocol = protocol.replace("_compatible", "") + obfs = obfs.replace("_compatible", "") + link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, base64.urlsafe_b64encode(user['passwd'])) + return "ssr://" + base64.urlsafe_b64encode(link) + def userinfo(self, user): ret = "" for key in user.keys(): @@ -38,15 +48,16 @@ def userinfo(self, user): ret += " %s : %s" % (key, val) elif val / 1024**2 < 4: val /= float(1024) - ret += " %s : %s K bytes" % (key, val) + ret += " %s : %s K Bytes" % (key, val) elif val / 1024**3 < 4: val /= float(1024**2) - ret += " %s : %s M bytes" % (key, val) + ret += " %s : %s M Bytes" % (key, val) else: val /= float(1024**3) - ret += " %s : %s G bytes" % (key, val) + ret += " %s : %s G Bytes" % (key, val) else: ret += " %s : %s" % (key, user[key]) + ret += "\n " + self.ssrlink(user) return ret def rand_pass(self): From cd6dab188c66e77935d86ed8e43106dc388e3bfa Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 8 Jul 2016 11:13:18 +0800 Subject: [PATCH 225/487] avoid making mudb.json empty test more encryptor add rc4-md5-6 --- db_transfer.py | 3 ++- mujson_mgr.py | 9 +++++---- shadowsocks/crypto/rc4_md5.py | 1 + shadowsocks/encrypt_test.py | 30 ++++++++++++++++++++++++------ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 0da6fe4d..7ed02c46 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -250,8 +250,9 @@ def update_all_user(self, dt_transfer): if rows: output = json.dumps(rows, sort_keys=True, indent=4, separators=(',', ': ')) - with open(config_path, 'w') as f: + with open(config_path, 'r+') as f: f.write(output) + f.truncate() def pull_db_all_user(self): import json diff --git a/mujson_mgr.py b/mujson_mgr.py index ff99ac23..741c44c5 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -2,7 +2,7 @@ # -*- coding: UTF-8 -*- import traceback -from shadowsocks import shell +from shadowsocks import shell, common from configloader import load_config, get_config import random import getopt @@ -21,8 +21,9 @@ def load(self, path): def save(self, path): if self.json: output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) - with open(path, 'w') as f: + with open(path, 'r+') as f: f.write(output) + f.truncate() class MuMgr(object): def __init__(self): @@ -35,8 +36,8 @@ def ssrlink(self, user): obfs = user.get('obfs', '') protocol = protocol.replace("_compatible", "") obfs = obfs.replace("_compatible", "") - link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, base64.urlsafe_b64encode(user['passwd'])) - return "ssr://" + base64.urlsafe_b64encode(link) + link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd'])))) + return "ssr://" + common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))) def userinfo(self, user): ret = "" diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index 1f07a82f..b0105ec2 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -35,6 +35,7 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, ciphers = { 'rc4-md5': (16, 16, create_cipher), + 'rc4-md5-6': (16, 6, create_cipher), } diff --git a/shadowsocks/encrypt_test.py b/shadowsocks/encrypt_test.py index 0121d633..d5e50778 100644 --- a/shadowsocks/encrypt_test.py +++ b/shadowsocks/encrypt_test.py @@ -12,6 +12,18 @@ from shadowsocks.crypto import sodium from shadowsocks.crypto import table +def run(func): + try: + func() + except: + pass + +def run_n(func, name): + try: + func(name) + except: + pass + def main(): print("\n""rc4_md5") rc4_md5.test() @@ -19,14 +31,20 @@ def main(): openssl.test_aes_256_cfb() print("\n""aes-128-cfb") openssl.test_aes_128_cfb() - print("\n""rc4") - openssl.test_rc4() + print("\n""bf-cfb") + run(openssl.test_bf_cfb) + print("\n""camellia-128-cfb") + run_n(openssl.run_method, "camellia-128-cfb") + print("\n""cast5-cfb") + run_n(openssl.run_method, "cast5-cfb") + print("\n""idea-cfb") + run_n(openssl.run_method, "idea-cfb") + print("\n""seed-cfb") + run_n(openssl.run_method, "seed-cfb") print("\n""salsa20") - sodium.test_salsa20() + run(sodium.test_salsa20) print("\n""chacha20") - sodium.test_chacha20() - print("\n""table") - table.test_encryption() + run(sodium.test_chacha20) if __name__ == '__main__': main() From e45d5d308e2c6c37ed36677bb4fa1eabaf116704 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 9 Jul 2016 12:58:46 +0800 Subject: [PATCH 226/487] custom tls time diff. not check time diff by default --- shadowsocks/obfsplugin/obfs_tls.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index f483d46d..529444ee 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -281,7 +281,7 @@ def __init__(self, method): self.send_buffer = b'' self.recv_buffer = b'' self.client_id = b'' - self.max_time_dif = 60 * 60 # time dif (second) setting + self.max_time_dif = 0 # time dif (second) setting self.tls_version = b'\x03\x03' def init_data(self): @@ -457,8 +457,13 @@ def server_decode(self, buf): sha1 = hmac.new(self.server_info.key + sessionid, verifyid[:22], hashlib.sha1).digest()[:10] utc_time = struct.unpack('>I', verifyid[:4])[0] time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time) - if time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ - or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: + if self.server_info.obfs_param: + try: + self.max_time_dif = int(self.server_info.obfs_param) + except: + pass + if self.max_time_dif > 0 and (time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ + or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2): logging.info("tls_auth wrong time") return self.decode_error_return(ogn_buf) if sha1 != verifyid[22:]: From b812ab10bfb0a50ac416189f9161da2d4b220509 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 9 Jul 2016 13:56:05 +0800 Subject: [PATCH 227/487] fix reload module --- importloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/importloader.py b/importloader.py index fb7c26b7..c917cb7d 100644 --- a/importloader.py +++ b/importloader.py @@ -4,6 +4,7 @@ def load(name): try: obj = __import__(name) + reload(obj) return obj except: pass @@ -11,6 +12,7 @@ def load(name): try: import importlib obj = importlib.__import__(name) + importlib.reload(obj) return obj except: pass From 783267dba04ffa2d234a9cd5a80219a1c745ef69 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 15 Jul 2016 19:16:02 +0800 Subject: [PATCH 228/487] support sspanelv3 node info & logs --- db_transfer.py | 103 ++++++++++++++++++++++++++++++++++++++++++++----- server.py | 4 +- switchrule.py | 4 +- 3 files changed, 98 insertions(+), 13 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 7ed02c46..502aef6a 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -18,6 +18,8 @@ def __init__(self): import threading self.last_get_transfer = {} self.event = threading.Event() + self.user_pass = {} + self.port_uid_table = {} def update_all_user(self, dt_transfer): import cymysql @@ -27,8 +29,6 @@ def update_all_user(self, dt_transfer): query_sub_in = None last_time = time.time() for id in dt_transfer.keys(): - if dt_transfer[id][0] == 0 and dt_transfer[id][1] == 0: - continue query_sub_when += ' WHEN %s THEN u+%s' % (id, dt_transfer[id][0]) query_sub_when2 += ' WHEN %s THEN d+%s' % (id, dt_transfer[id][1]) if query_sub_in is not None: @@ -41,7 +41,6 @@ def update_all_user(self, dt_transfer): ' END, d = CASE port' + query_sub_when2 + \ ' END, t = ' + str(int(last_time)) + \ ' WHERE port IN (%s)' % query_sub_in - #print query_sql conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') cur = conn.cursor() @@ -57,23 +56,26 @@ def push_db_all_user(self): #上次和本次的增量 dt_transfer = {} for id in curr_transfer.keys(): + update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) if id in last_transfer: - if last_transfer[id][0] == curr_transfer[id][0] and last_transfer[id][1] == curr_transfer[id][1]: - continue - elif curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + if curr_transfer[id][0] + curr_transfer[id][1] - last_transfer[id][0] - last_transfer[id][1] < update_trs: + self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue - elif last_transfer[id][0] <= curr_transfer[id][0] and \ - last_transfer[id][1] <= curr_transfer[id][1]: + if last_transfer[id][0] <= curr_transfer[id][0] and \ + last_transfer[id][1] <= curr_transfer[id][1]: dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * get_config().TRANSFER_MUL), int((curr_transfer[id][1] - last_transfer[id][1]) * get_config().TRANSFER_MUL)] else: dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] else: - if curr_transfer[id][0] == 0 and curr_transfer[id][1] == 0: + if curr_transfer[id][0] + curr_transfer[id][1] < update_trs: + self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] + if id in self.user_pass: + del self.user_pass[id] self.update_all_user(dt_transfer) self.last_get_transfer = curr_transfer @@ -126,8 +128,9 @@ def del_server_out_of_bound_safe(self, last_rows, rows): port = row['port'] passwd = common.to_bytes(row['passwd']) cfg = {'password': passwd} + self.port_uid_table[row['port']] = row['id'] - read_config_keys = ['method', 'obfs', 'protocol', 'forbidden_ip', 'forbidden_port'] + read_config_keys = ['method', 'obfs', 'obfs_param', 'protocol', 'protocol_param', 'forbidden_ip', 'forbidden_port'] for name in read_config_keys: if name in row and row[name]: cfg[name] = row[name] @@ -178,6 +181,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): else: logging.info('db stop server at port [%s] reason: port not exist' % (row['port'])) ServerPool.get_instance().cb_del_server(row['port']) + del self.port_uid_table[row['port']] if len(new_servers) > 0: from shadowsocks import eventloop @@ -230,6 +234,85 @@ def thread_db_stop(): global db_instance db_instance.event.set() +class Dbv3Transfer(DbTransfer): + def __init__(self): + super(Dbv3Transfer, self).__init__() + + def update_all_user(self, dt_transfer): + import cymysql + + query_head = 'UPDATE user' + query_sub_when = '' + query_sub_when2 = '' + query_sub_in = None + last_time = time.time() + + alive_user_count = 0 + bandwidth_thistime = 0 + + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, + passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') + conn.autocommit(True) + + for id in dt_transfer.keys(): + transfer = dt_transfer[id] + query_sub_when += ' WHEN %s THEN u+%s' % (id, transfer[0]) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, transfer[1]) + + cur = conn.cursor() + cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `Node_ID`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ + str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ + str(get_config().NODE_ID) + "', '" + str(get_config().TRANSFER_MUL) + "', '" + \ + self.traffic_format(transfer[0] + transfer[1]) + "', unix_timestamp()); ") + cur.close() + + alive_user_count = alive_user_count + 1 + bandwidth_thistime = bandwidth_thistime + transfer[0] + transfer[1] + + if query_sub_in is not None: + query_sub_in += ',%s' % id + else: + query_sub_in = '%s' % id + + if query_sub_when != '': + query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ + ' END, d = CASE port' + query_sub_when2 + \ + ' END, t = ' + str(int(last_time)) + \ + ' WHERE port IN (%s)' % query_sub_in + cur = conn.cursor() + cur.execute(query_sql) + cur.close() + + cur = conn.cursor() + cur.execute("INSERT INTO `ss_node_online_log` (`id`, `Node_ID`, `online_user`, `log_time`) VALUES (NULL, '" + \ + str(get_config().NODE_ID) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") + cur.close() + + cur = conn.cursor() + cur.execute("INSERT INTO `ss_node_info_log` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ + str(get_config().NODE_ID) + "', '" + str(self.uptime()) + "', '" + \ + str(self.load()) + "', unix_timestamp()); ") + cur.close() + + conn.close() + + def load(self): + import os + return os.popen("cat /proc/loadavg | awk '{ print $1\" \"$2\" \"$3 }'").readlines()[0] + + def uptime(self): + with open('/proc/uptime', 'r') as f: + return float(f.readline().split()[0]) + + def traffic_format(self, traffic): + if traffic < 1024 * 8: + return str(traffic) + "B"; + + if traffic < 1024 * 1024 * 8: + return str(round((traffic / 1024.0), 2)) + "KB"; + + return str(round((traffic / 1048576.0), 2)) + "MB"; + class MuJsonTransfer(DbTransfer): def __init__(self): super(MuJsonTransfer, self).__init__() diff --git a/server.py b/server.py index 75ce3401..4636721e 100644 --- a/server.py +++ b/server.py @@ -47,8 +47,10 @@ def main(): else: if get_config().API_INTERFACE == 'mudbjson': thread = MainThread(db_transfer.MuJsonTransfer) - else: + elif get_config().API_INTERFACE == 'sspanelv2': thread = MainThread(db_transfer.DbTransfer) + else: + thread = MainThread(db_transfer.Dbv3Transfer) thread.start() try: while thread.is_alive(): diff --git a/switchrule.py b/switchrule.py index 5b2d3133..d67c6c6a 100644 --- a/switchrule.py +++ b/switchrule.py @@ -3,9 +3,9 @@ def getKeys(): key_list = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] if get_config().API_INTERFACE == 'sspanelv3': - key_list += ['method'] + key_list += ['id', 'method'] elif get_config().API_INTERFACE == 'sspanelv3ssr': - key_list += ['method', 'obfs', 'protocol'] + key_list += ['id', 'method', 'obfs', 'protocol'] return key_list #return key_list + ['plan'] # append the column name 'plan' From 6255a1d32122c3dd73a64e421ad240d610321cea Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 15 Jul 2016 23:01:53 +0800 Subject: [PATCH 229/487] fix mudbjson mode --- db_transfer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 502aef6a..5f93c7b7 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -128,7 +128,8 @@ def del_server_out_of_bound_safe(self, last_rows, rows): port = row['port'] passwd = common.to_bytes(row['passwd']) cfg = {'password': passwd} - self.port_uid_table[row['port']] = row['id'] + if 'id' in row: + self.port_uid_table[row['port']] = row['id'] read_config_keys = ['method', 'obfs', 'obfs_param', 'protocol', 'protocol_param', 'forbidden_ip', 'forbidden_port'] for name in read_config_keys: @@ -181,7 +182,8 @@ def del_server_out_of_bound_safe(self, last_rows, rows): else: logging.info('db stop server at port [%s] reason: port not exist' % (row['port'])) ServerPool.get_instance().cb_del_server(row['port']) - del self.port_uid_table[row['port']] + if row['port'] in self.port_uid_table: + del self.port_uid_table[row['port']] if len(new_servers) > 0: from shadowsocks import eventloop @@ -260,7 +262,8 @@ def update_all_user(self, dt_transfer): query_sub_when2 += ' WHEN %s THEN d+%s' % (id, transfer[1]) cur = conn.cursor() - cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `Node_ID`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ + if id in self.port_uid_table: + cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `Node_ID`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ str(get_config().NODE_ID) + "', '" + str(get_config().TRANSFER_MUL) + "', '" + \ self.traffic_format(transfer[0] + transfer[1]) + "', unix_timestamp()); ") From d8dda5042f9a5ffe11ca1c2d30ce58cbcbfa9bda Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 16 Jul 2016 18:26:04 +0800 Subject: [PATCH 230/487] change online user counter & note traffic --- db_transfer.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 5f93c7b7..4a7d716b 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -29,12 +29,18 @@ def update_all_user(self, dt_transfer): query_sub_in = None last_time = time.time() for id in dt_transfer.keys(): - query_sub_when += ' WHEN %s THEN u+%s' % (id, dt_transfer[id][0]) - query_sub_when2 += ' WHEN %s THEN d+%s' % (id, dt_transfer[id][1]) + transfer = dt_transfer[id] + update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) + if transfer[0] + transfer[1] < update_trs: + continue + + query_sub_when += ' WHEN %s THEN u+%s' % (id, transfer[0]) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, transfer[1]) if query_sub_in is not None: query_sub_in += ',%s' % id else: query_sub_in = '%s' % id + if query_sub_when == '': return query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ @@ -56,9 +62,8 @@ def push_db_all_user(self): #上次和本次的增量 dt_transfer = {} for id in curr_transfer.keys(): - update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) if id in last_transfer: - if curr_transfer[id][0] + curr_transfer[id][1] - last_transfer[id][0] - last_transfer[id][1] < update_trs: + if curr_transfer[id][0] + curr_transfer[id][1] - last_transfer[id][0] - last_transfer[id][1] <= 0: self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue if last_transfer[id][0] <= curr_transfer[id][0] and \ @@ -69,7 +74,7 @@ def push_db_all_user(self): dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] else: - if curr_transfer[id][0] + curr_transfer[id][1] < update_trs: + if curr_transfer[id][0] + curr_transfer[id][1] <= 0: self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), @@ -258,6 +263,13 @@ def update_all_user(self, dt_transfer): for id in dt_transfer.keys(): transfer = dt_transfer[id] + alive_user_count = alive_user_count + 1 + bandwidth_thistime = bandwidth_thistime + transfer[0] + transfer[1] + + update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) + if transfer[0] + transfer[1] < update_trs: + continue + query_sub_when += ' WHEN %s THEN u+%s' % (id, transfer[0]) query_sub_when2 += ' WHEN %s THEN d+%s' % (id, transfer[1]) @@ -269,9 +281,6 @@ def update_all_user(self, dt_transfer): self.traffic_format(transfer[0] + transfer[1]) + "', unix_timestamp()); ") cur.close() - alive_user_count = alive_user_count + 1 - bandwidth_thistime = bandwidth_thistime + transfer[0] + transfer[1] - if query_sub_in is not None: query_sub_in += ',%s' % id else: From 4fdd9ea0ab9c0bc5e03669bd5a3340b23040c643 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 18 Jul 2016 12:56:16 +0800 Subject: [PATCH 231/487] fix transfer update --- db_transfer.py | 77 ++++++++++++++++++++++++++++++-------------------- mujson_mgr.py | 8 +++++- switchrule.py | 2 +- 3 files changed, 55 insertions(+), 32 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 4a7d716b..d466a31d 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -23,19 +23,26 @@ def __init__(self): def update_all_user(self, dt_transfer): import cymysql + update_transfer = {} + query_head = 'UPDATE user' query_sub_when = '' query_sub_when2 = '' query_sub_in = None last_time = time.time() + for id in dt_transfer.keys(): transfer = dt_transfer[id] update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) if transfer[0] + transfer[1] < update_trs: continue + if id in self.user_pass: + del self.user_pass[id] + + query_sub_when += ' WHEN %s THEN u+%s' % (id, int(transfer[0] * get_config().TRANSFER_MUL)) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, int(transfer[1] * get_config().TRANSFER_MUL)) + update_transfer[id] = transfer - query_sub_when += ' WHEN %s THEN u+%s' % (id, transfer[0]) - query_sub_when2 += ' WHEN %s THEN d+%s' % (id, transfer[1]) if query_sub_in is not None: query_sub_in += ',%s' % id else: @@ -64,26 +71,22 @@ def push_db_all_user(self): for id in curr_transfer.keys(): if id in last_transfer: if curr_transfer[id][0] + curr_transfer[id][1] - last_transfer[id][0] - last_transfer[id][1] <= 0: - self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue if last_transfer[id][0] <= curr_transfer[id][0] and \ last_transfer[id][1] <= curr_transfer[id][1]: - dt_transfer[id] = [int((curr_transfer[id][0] - last_transfer[id][0]) * get_config().TRANSFER_MUL), - int((curr_transfer[id][1] - last_transfer[id][1]) * get_config().TRANSFER_MUL)] + dt_transfer[id] = [curr_transfer[id][0] - last_transfer[id][0], + curr_transfer[id][1] - last_transfer[id][1]] else: - dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), - int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] + dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] else: if curr_transfer[id][0] + curr_transfer[id][1] <= 0: - self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue - dt_transfer[id] = [int(curr_transfer[id][0] * get_config().TRANSFER_MUL), - int(curr_transfer[id][1] * get_config().TRANSFER_MUL)] - if id in self.user_pass: - del self.user_pass[id] + dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] - self.update_all_user(dt_transfer) - self.last_get_transfer = curr_transfer + update_transfer = self.update_all_user(dt_transfer) + for id in update_transfer.keys(): + last = self.last_get_transfer.get(id, [0,0]) + self.last_get_transfer[id] = [last[0] + update_transfer[id][0], last[1] + update_transfer[id][1]] def pull_db_all_user(self): import cymysql @@ -247,6 +250,7 @@ def __init__(self): def update_all_user(self, dt_transfer): import cymysql + update_transfer = {} query_head = 'UPDATE user' query_sub_when = '' @@ -268,17 +272,24 @@ def update_all_user(self, dt_transfer): update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) if transfer[0] + transfer[1] < update_trs: + self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue + if id in self.user_pass: + del self.user_pass[id] - query_sub_when += ' WHEN %s THEN u+%s' % (id, transfer[0]) - query_sub_when2 += ' WHEN %s THEN d+%s' % (id, transfer[1]) + query_sub_when += ' WHEN %s THEN u+%s' % (id, int(transfer[0] * get_config().TRANSFER_MUL)) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, int(transfer[1] * get_config().TRANSFER_MUL)) + update_transfer[id] = transfer cur = conn.cursor() - if id in self.port_uid_table: - cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `Node_ID`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ - str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ - str(get_config().NODE_ID) + "', '" + str(get_config().TRANSFER_MUL) + "', '" + \ - self.traffic_format(transfer[0] + transfer[1]) + "', unix_timestamp()); ") + try: + if id in self.port_uid_table: + cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `Node_ID`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ + str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ + str(get_config().NODE_ID) + "', '" + str(get_config().TRANSFER_MUL) + "', '" + \ + self.traffic_format(transfer[0] + transfer[1]) + "', unix_timestamp()); ") + except: + logging.warn('no `user_traffic_log` in db') cur.close() if query_sub_in is not None: @@ -295,18 +306,22 @@ def update_all_user(self, dt_transfer): cur.execute(query_sql) cur.close() - cur = conn.cursor() - cur.execute("INSERT INTO `ss_node_online_log` (`id`, `Node_ID`, `online_user`, `log_time`) VALUES (NULL, '" + \ - str(get_config().NODE_ID) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") - cur.close() + try: + cur = conn.cursor() + cur.execute("INSERT INTO `ss_node_online_log` (`id`, `Node_ID`, `online_user`, `log_time`) VALUES (NULL, '" + \ + str(get_config().NODE_ID) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") + cur.close() - cur = conn.cursor() - cur.execute("INSERT INTO `ss_node_info_log` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ - str(get_config().NODE_ID) + "', '" + str(self.uptime()) + "', '" + \ - str(self.load()) + "', unix_timestamp()); ") - cur.close() + cur = conn.cursor() + cur.execute("INSERT INTO `ss_node_info_log` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ + str(get_config().NODE_ID) + "', '" + str(self.uptime()) + "', '" + \ + str(self.load()) + "', unix_timestamp()); ") + cur.close() + except: + logging.warn('no `ss_node_online_log` or `ss_node_info_log` in db') conn.close() + return update_transfer def load(self): import os @@ -349,6 +364,8 @@ def update_all_user(self, dt_transfer): f.write(output) f.truncate() + return dt_transfer + def pull_db_all_user(self): import json rows = None diff --git a/mujson_mgr.py b/mujson_mgr.py index 741c44c5..13142d7d 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -163,6 +163,8 @@ def print_server_help(): -m METHOD encryption method, default: aes-128-cfb -O PROTOCOL protocol plugin, default: auth_sha1_v2_compatible -o OBFS obfs plugin, default: tls1.2_ticket_auth_compatible + -G PROTOCOL_PARAM protocol plugin param + -g OBFS_PARAM obfs plugin param -t TRANSFER max transfer for G bytes, default: 1048576, can be float point number -f FORBID set forbidden ports. Example (ban 1~79 and 81~100): -f "1-79,81-100" @@ -171,7 +173,7 @@ def print_server_help(): ''') def main(): - shortopts = 'adeclu:p:k:O:o:m:t:f:h' + shortopts = 'adeclu:p:k:O:o:G:g:m:t:f:h' longopts = ['help'] action = None user = {} @@ -198,6 +200,10 @@ def main(): user['obfs'] = value elif key == '-O': user['protocol'] = value + elif key == '-g': + user['obfs_param'] = value + elif key == '-G': + user['protocol_param'] = value elif key == '-m': user['method'] = value elif key == '-f': diff --git a/switchrule.py b/switchrule.py index d67c6c6a..571a19f0 100644 --- a/switchrule.py +++ b/switchrule.py @@ -5,7 +5,7 @@ def getKeys(): if get_config().API_INTERFACE == 'sspanelv3': key_list += ['id', 'method'] elif get_config().API_INTERFACE == 'sspanelv3ssr': - key_list += ['id', 'method', 'obfs', 'protocol'] + key_list += ['id', 'method', 'obfs', 'protocol', 'obfs_param', 'protocol_param'] return key_list #return key_list + ['plan'] # append the column name 'plan' From 1dbc5e780da779fc6d0f4d47a56d2a5f8ce78be9 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 18 Jul 2016 15:40:41 +0800 Subject: [PATCH 232/487] fix column name --- db_transfer.py | 4 ++-- shadowsocks.sql | 24 ------------------------ 2 files changed, 2 insertions(+), 26 deletions(-) delete mode 100644 shadowsocks.sql diff --git a/db_transfer.py b/db_transfer.py index d466a31d..36a37c41 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -284,7 +284,7 @@ def update_all_user(self, dt_transfer): cur = conn.cursor() try: if id in self.port_uid_table: - cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `Node_ID`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ + cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `node_id`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ str(get_config().NODE_ID) + "', '" + str(get_config().TRANSFER_MUL) + "', '" + \ self.traffic_format(transfer[0] + transfer[1]) + "', unix_timestamp()); ") @@ -308,7 +308,7 @@ def update_all_user(self, dt_transfer): try: cur = conn.cursor() - cur.execute("INSERT INTO `ss_node_online_log` (`id`, `Node_ID`, `online_user`, `log_time`) VALUES (NULL, '" + \ + cur.execute("INSERT INTO `ss_node_online_log` (`id`, `node_id`, `online_user`, `log_time`) VALUES (NULL, '" + \ str(get_config().NODE_ID) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") cur.close() diff --git a/shadowsocks.sql b/shadowsocks.sql deleted file mode 100644 index 8d84992d..00000000 --- a/shadowsocks.sql +++ /dev/null @@ -1,24 +0,0 @@ -SET FOREIGN_KEY_CHECKS=0; - -CREATE TABLE `user` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `email` varchar(32) NOT NULL, - `pass` varchar(16) NOT NULL, - `passwd` varchar(16) NOT NULL, - `t` int(11) NOT NULL DEFAULT '0', - `u` bigint(20) NOT NULL, - `d` bigint(20) NOT NULL, - `transfer_enable` bigint(20) NOT NULL, - `port` int(11) NOT NULL, - `switch` tinyint(4) NOT NULL DEFAULT '1', - `enable` tinyint(4) NOT NULL DEFAULT '1', - `type` tinyint(4) NOT NULL DEFAULT '1', - `last_get_gift_time` int(11) NOT NULL DEFAULT '0', - `last_rest_pass_time` int(11) NOT NULL DEFAULT '0', - PRIMARY KEY (`id`,`port`) -) ENGINE=InnoDB AUTO_INCREMENT=415 DEFAULT CHARSET=utf8; - --- ---------------------------- --- Records of user --- ---------------------------- -INSERT INTO `user` VALUES ('7', 'test@test.com', '123456', '0000000', '1410609560', '0', '0', '9320666234', '50000', '1', '1', '7', '0', '0'); \ No newline at end of file From 68d0c8e2607ccdc3573a60673ac5f01d253d73b2 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 18 Jul 2016 16:06:56 +0800 Subject: [PATCH 233/487] auto get ip in mujson_mgr.py --- mujson_mgr.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mujson_mgr.py b/mujson_mgr.py index 13142d7d..14594981 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -30,6 +30,12 @@ def __init__(self): self.config_path = get_config().MUDB_FILE self.server_addr = get_config().SERVER_PUB_ADDR self.data = MuJsonLoader() + if self.server_addr == '127.0.0.1': + try: + import socket + self.server_addr = socket.gethostbyname(socket.gethostname()) + except: + pass def ssrlink(self, user): protocol = user.get('protocol', '') From 5709f7d9018254eb0c55a74966edd06b430cd2a6 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 18 Jul 2016 18:23:49 +0800 Subject: [PATCH 234/487] fix a bug --- db_transfer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index 36a37c41..ad531ed5 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -49,7 +49,7 @@ def update_all_user(self, dt_transfer): query_sub_in = '%s' % id if query_sub_when == '': - return + return update_transfer query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ ' END, d = CASE port' + query_sub_when2 + \ ' END, t = ' + str(int(last_time)) + \ @@ -61,6 +61,7 @@ def update_all_user(self, dt_transfer): cur.close() conn.commit() conn.close() + return update_transfer def push_db_all_user(self): #更新用户流量到数据库 From 6db1445d2a5fc29a317e757a96d846ced83c2113 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 12:50:07 +0800 Subject: [PATCH 235/487] auto get ip, ignore 'enable' --- mujson_mgr.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 14594981..268961c3 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -28,26 +28,45 @@ def save(self, path): class MuMgr(object): def __init__(self): self.config_path = get_config().MUDB_FILE - self.server_addr = get_config().SERVER_PUB_ADDR + try: + self.server_addr = get_config().SERVER_PUB_ADDR + except: + self.server_addr = '127.0.0.1' self.data = MuJsonLoader() + if self.server_addr == '127.0.0.1': + self.server_addr = self.getipaddr() + + def getipaddr(self, ifname = 'eth0'): + import socket + import struct + import fcntl + ret = '127.0.0.1' + try: + ret = socket.gethostbyname(socket.getfqdn(socket.gethostname())) + except: + pass + if ret == '127.0.0.1': try: - import socket - self.server_addr = socket.gethostbyname(socket.gethostname()) + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ret = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24]) except: pass + return ret - def ssrlink(self, user): + def ssrlink(self, user, encode): protocol = user.get('protocol', '') obfs = user.get('obfs', '') protocol = protocol.replace("_compatible", "") obfs = obfs.replace("_compatible", "") link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd'])))) - return "ssr://" + common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))) + return "ssr://" + ( encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))) or link) def userinfo(self, user): ret = "" for key in user.keys(): + if key in ['enable']: + continue ret += '\n' if key in ['transfer_enable', 'u', 'd'] : val = user[key] @@ -64,7 +83,8 @@ def userinfo(self, user): ret += " %s : %s G Bytes" % (key, val) else: ret += " %s : %s" % (key, user[key]) - ret += "\n " + self.ssrlink(user) + ret += "\n " + self.ssrlink(user, False) + ret += "\n " + self.ssrlink(user, True) return ret def rand_pass(self): From 8e73f50ff5f173e46b51aef2fc60e228c28290fd Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 15:23:35 +0800 Subject: [PATCH 236/487] fix http_post & http_simple redirect --- shadowsocks/obfsplugin/http_simple.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 00568670..6783a32e 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -175,9 +175,7 @@ def server_decode(self, buf): if len(ret_buf) >= 7: self.has_recv_header = True return (ret_buf, True, False) - if len(ret_buf) == 0: - return self.not_match_return(buf) - return (b'', True, False) + return self.not_match_return(buf) else: return (b'', True, False) @@ -244,9 +242,7 @@ def server_decode(self, buf): if len(ret_buf) >= 7: self.has_recv_header = True return (ret_buf, True, False) - if len(ret_buf) == 0: - return self.not_match_return(buf) - return (b'', True, False) + return self.not_match_return(buf) else: return (b'', True, False) From 3fd0fbf1d0206536a545012717e5d4a88de2e3ef Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 15:26:17 +0800 Subject: [PATCH 237/487] mysql SSL connect mujson_mgr add obfs & protocol mapping --- apiconfig.py | 5 +++++ db_transfer.py | 33 +++++++++++++++++++++++++++------ mujson_mgr.py | 24 ++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/apiconfig.py b/apiconfig.py index b69c50eb..6576e7e5 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -15,6 +15,11 @@ MYSQL_DB = 'shadowsocks' MYSQL_UPDATE_TIME = 60 +MYSQL_SSL_ENABLE = 0 +MYSQL_SSL_CA = '' +MYSQL_SSL_CERT = '' +MYSQL_SSL_KEY = '' + # API API_HOST = '127.0.0.1' API_PORT = 80 diff --git a/db_transfer.py b/db_transfer.py index ad531ed5..743f88ec 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -54,8 +54,15 @@ def update_all_user(self, dt_transfer): ' END, d = CASE port' + query_sub_when2 + \ ' END, t = ' + str(int(last_time)) + \ ' WHERE port IN (%s)' % query_sub_in - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, - passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') + if get_config().MYSQL_SSL_ENABLE == 1: + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, + user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, + db=get_config().MYSQL_DB, charset='utf8', + ssl={'ca':get_config().MYSQL_SSL_CA,'cert':get_config().MYSQL_SSL_CERT,'key':get_config().MYSQL_SSL_KEY}) + else: + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, + user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, + db=get_config().MYSQL_DB, charset='utf8') cur = conn.cursor() cur.execute(query_sql) cur.close() @@ -97,8 +104,15 @@ def pull_db_all_user(self): keys = switchrule.getKeys() except Exception as e: keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, - passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') + if get_config().MYSQL_SSL_ENABLE == 1: + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, + user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, + db=get_config().MYSQL_DB, charset='utf8', + ssl={'ca':get_config().MYSQL_SSL_CA,'cert':get_config().MYSQL_SSL_CERT,'key':get_config().MYSQL_SSL_KEY}) + else: + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, + user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, + db=get_config().MYSQL_DB, charset='utf8') cur = conn.cursor() cur.execute("SELECT " + ','.join(keys) + " FROM user") rows = [] @@ -262,8 +276,15 @@ def update_all_user(self, dt_transfer): alive_user_count = 0 bandwidth_thistime = 0 - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, user=get_config().MYSQL_USER, - passwd=get_config().MYSQL_PASS, db=get_config().MYSQL_DB, charset='utf8') + if get_config().MYSQL_SSL_ENABLE == 1: + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, + user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, + db=get_config().MYSQL_DB, charset='utf8', + ssl={'ca':get_config().MYSQL_SSL_CA,'cert':get_config().MYSQL_SSL_CERT,'key':get_config().MYSQL_SSL_KEY}) + else: + conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, + user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, + db=get_config().MYSQL_DB, charset='utf8') conn.autocommit(True) for id in dt_transfer.keys(): diff --git a/mujson_mgr.py b/mujson_mgr.py index 268961c3..a9cf0f5f 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -203,6 +203,20 @@ def main(): longopts = ['help'] action = None user = {} + fast_set_obfs = {'0': 'plain', + '1': 'http_simple_compatible', + '-1': 'http_simple', + '2': 'http_post_compatible', + '-2': 'http_post', + '3': 'tls1.2_ticket_auth_compatible', + '-3': 'tls1.2_ticket_auth'} + fast_set_protocol = {'0': 'origin', + '1': 'verify_sha1_compatible', + '-1': 'verify_sha1', + '2': 'auth_sha1_compatible', + '-2': 'auth_sha1', + '3': 'auth_sha1_v2_compatible', + '-3': 'auth_sha1_v2'} try: optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) for key, value in optlist: @@ -223,9 +237,15 @@ def main(): elif key == '-k': user['passwd'] = value elif key == '-o': - user['obfs'] = value + if value in fast_set_obfs: + user['obfs'] = fast_set_obfs[value] + else: + user['obfs'] = value elif key == '-O': - user['protocol'] = value + if value in fast_set_protocol: + user['protocol'] = fast_set_protocol[value] + else: + user['protocol'] = value elif key == '-g': user['obfs_param'] = value elif key == '-G': From e2bae5874cc63ec171c0a47c9a61d85ed01d3bf7 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 15:43:02 +0800 Subject: [PATCH 238/487] catch decode error --- shadowsocks/tcprelay.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e1898fc7..387a2bb1 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -468,10 +468,16 @@ def _handle_stage_addr(self, ogn_data, data): data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) connecttype, remote_addr, remote_port, header_length = header_result - common.connect_log('%s connecting %s:%d from %s:%d' % + try: + common.connect_log('%s connecting %s:%d from %s:%d' % ((connecttype == 0) and 'TCP' or 'UDP', common.to_str(remote_addr), remote_port, self._client_address[0], self._client_address[1])) + except Exception as e: + common.connect_log('%s connecting %s:%d from %s:%d' % + ((connecttype == 0) and 'TCP' or 'UDP', + binascii.hexlify(remote_addr), remote_port, + self._client_address[0], self._client_address[1])) self._remote_address = (common.to_str(remote_addr), remote_port) self._remote_udp = (connecttype != 0) # pause reading From 3f6f16e4d6b95604fe7ff87c8add1a1d86d04f89 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 16:14:19 +0800 Subject: [PATCH 239/487] change the counter of the online users --- db_transfer.py | 20 ++++++++++++++------ shadowsocks/tcprelay.py | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 743f88ec..86fb5ceb 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -6,7 +6,7 @@ import sys from server_pool import ServerPool import traceback -from shadowsocks import common, shell +from shadowsocks import common, shell, lru_cache from configloader import load_config, get_config import importloader @@ -17,9 +17,11 @@ class DbTransfer(object): def __init__(self): import threading self.last_get_transfer = {} + self.last_update_transfer = {} self.event = threading.Event() self.user_pass = {} self.port_uid_table = {} + self.onlineuser_cache = lru_cache.LRUCache(timeout=60*30) def update_all_user(self, dt_transfer): import cymysql @@ -72,7 +74,7 @@ def update_all_user(self, dt_transfer): def push_db_all_user(self): #更新用户流量到数据库 - last_transfer = self.last_get_transfer + last_transfer = self.last_update_transfer curr_transfer = ServerPool.get_instance().get_servers_transfer() #上次和本次的增量 dt_transfer = {} @@ -90,11 +92,18 @@ def push_db_all_user(self): if curr_transfer[id][0] + curr_transfer[id][1] <= 0: continue dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + if id in self.last_get_transfer: + if curr_transfer[id][0] + curr_transfer[id][1] > self.last_get_transfer[id][0] + self.last_get_transfer[id][1]: + self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] + else: + self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] + self.onlineuser_cache.sweep() update_transfer = self.update_all_user(dt_transfer) for id in update_transfer.keys(): - last = self.last_get_transfer.get(id, [0,0]) - self.last_get_transfer[id] = [last[0] + update_transfer[id][0], last[1] + update_transfer[id][1]] + last = self.last_update_transfer.get(id, [0,0]) + self.last_update_transfer[id] = [last[0] + update_transfer[id][0], last[1] + update_transfer[id][1]] + self.last_get_transfer = curr_transfer def pull_db_all_user(self): import cymysql @@ -273,7 +282,7 @@ def update_all_user(self, dt_transfer): query_sub_in = None last_time = time.time() - alive_user_count = 0 + alive_user_count = len(self.onlineuser_cache) bandwidth_thistime = 0 if get_config().MYSQL_SSL_ENABLE == 1: @@ -289,7 +298,6 @@ def update_all_user(self, dt_transfer): for id in dt_transfer.keys(): transfer = dt_transfer[id] - alive_user_count = alive_user_count + 1 bandwidth_thistime = bandwidth_thistime + transfer[0] + transfer[1] update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 387a2bb1..c3e87890 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -370,7 +370,7 @@ def _handel_protocol_error(self, client_address, ogn_data): host, port = self._get_redirect_host(client_address, ogn_data) if port == 0: raise Exception('can not parse header') - data = b"\x03" + common.chr(len(host)) + common.to_bytes(host) + struct.pack('>H', port) + data = b"\x03" + common.to_bytes(common.chr(len(host))) + common.to_bytes(host) + struct.pack('>H', port) logging.warn("TCP data redir %s:%d %s" % (host, port, binascii.hexlify(data))) return data + ogn_data From 9538afebff03c4aa3d799d25c2819fdb07d423f8 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 16:41:42 +0800 Subject: [PATCH 240/487] check decode remote_addr --- shadowsocks/tcprelay.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c3e87890..557ef989 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -354,7 +354,7 @@ def _get_redirect_host(self, client_address, ogn_data): if type(host_list) == list: host_post = common.to_str(host_list[((hash_code & 0xffffffff) + addr) % len(host_list)]) else: - host_post = host_list + host_post = common.to_str(host_list) items = host_post.rsplit(':', 1) if len(items) > 1: try: @@ -464,20 +464,19 @@ def _handle_stage_addr(self, ogn_data, data): if data is None: data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) + if header_result is not None: + try: + common.to_str(remote_addr) + except Exception as e: + header_result = None if header_result is None: data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) connecttype, remote_addr, remote_port, header_length = header_result - try: - common.connect_log('%s connecting %s:%d from %s:%d' % + common.connect_log('%s connecting %s:%d from %s:%d' % ((connecttype == 0) and 'TCP' or 'UDP', common.to_str(remote_addr), remote_port, self._client_address[0], self._client_address[1])) - except Exception as e: - common.connect_log('%s connecting %s:%d from %s:%d' % - ((connecttype == 0) and 'TCP' or 'UDP', - binascii.hexlify(remote_addr), remote_port, - self._client_address[0], self._client_address[1])) self._remote_address = (common.to_str(remote_addr), remote_port) self._remote_udp = (connecttype != 0) # pause reading From 32957158b9dcee71a013a0b907431643db979faf Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 16:44:10 +0800 Subject: [PATCH 241/487] check decode remote_addr --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 557ef989..085565cb 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -466,7 +466,7 @@ def _handle_stage_addr(self, ogn_data, data): header_result = parse_header(data) if header_result is not None: try: - common.to_str(remote_addr) + common.to_str(header_result[1]) except Exception as e: header_result = None if header_result is None: From 794121f11171373a05b7291eaf1a86be7537b772 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Tue, 19 Jul 2016 21:29:07 +0800 Subject: [PATCH 242/487] show port details --- db_transfer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 86fb5ceb..f91cadce 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -205,7 +205,9 @@ def del_server_out_of_bound_safe(self, last_rows, rows): elif allow and ServerPool.get_instance().server_run_status(port) is False: #new_servers[port] = passwd - logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) + protocol = cfg.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) + obfs = cfg.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) + logging.info('db start server at port [%s] pass [%s] protocol [%s] obfs [%s]' % (port, passwd, protocol, obfs)) ServerPool.get_instance().new_server(port, cfg) for row in last_rows: @@ -222,7 +224,9 @@ def del_server_out_of_bound_safe(self, last_rows, rows): self.event.wait(eventloop.TIMEOUT_PRECISION + eventloop.TIMEOUT_PRECISION / 2) for port in new_servers.keys(): passwd, cfg = new_servers[port] - logging.info('db start server at port [%s] pass [%s]' % (port, passwd)) + protocol = cfb.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) + obfs = cfb.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) + logging.info('db start server at port [%s] pass [%s] protocol [%s] obfs [%s]' % (port, passwd, protocol, obfs)) ServerPool.get_instance().new_server(port, cfg) @staticmethod From 7bfafa7f3d0dfe38dc3ccaddabf9f860a1b49a34 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 20 Jul 2016 11:13:56 +0800 Subject: [PATCH 243/487] fix cfg --- db_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index f91cadce..f3892160 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -224,8 +224,8 @@ def del_server_out_of_bound_safe(self, last_rows, rows): self.event.wait(eventloop.TIMEOUT_PRECISION + eventloop.TIMEOUT_PRECISION / 2) for port in new_servers.keys(): passwd, cfg = new_servers[port] - protocol = cfb.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) - obfs = cfb.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) + protocol = cfg.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) + obfs = cfg.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) logging.info('db start server at port [%s] pass [%s] protocol [%s] obfs [%s]' % (port, passwd, protocol, obfs)) ServerPool.get_instance().new_server(port, cfg) From 9c80dee9dadf69c117f0e30145ee43eaa3d72137 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 21 Jul 2016 11:35:27 +0800 Subject: [PATCH 244/487] fix traffic & uptime --- db_transfer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index f3892160..a422aa11 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -321,7 +321,7 @@ def update_all_user(self, dt_transfer): cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `node_id`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ str(get_config().NODE_ID) + "', '" + str(get_config().TRANSFER_MUL) + "', '" + \ - self.traffic_format(transfer[0] + transfer[1]) + "', unix_timestamp()); ") + self.traffic_format((transfer[0] + transfer[1]) * get_config().TRANSFER_MUL) + "', unix_timestamp()); ") except: logging.warn('no `user_traffic_log` in db') cur.close() @@ -362,14 +362,13 @@ def load(self): return os.popen("cat /proc/loadavg | awk '{ print $1\" \"$2\" \"$3 }'").readlines()[0] def uptime(self): - with open('/proc/uptime', 'r') as f: - return float(f.readline().split()[0]) + return time.clock() def traffic_format(self, traffic): if traffic < 1024 * 8: return str(traffic) + "B"; - if traffic < 1024 * 1024 * 8: + if traffic < 1024 * 1024 * 2: return str(round((traffic / 1024.0), 2)) + "KB"; return str(round((traffic / 1048576.0), 2)) + "MB"; From c3484b337512815cd34b648869dfeec12cb9c147 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 21 Jul 2016 12:58:18 +0800 Subject: [PATCH 245/487] fix uptime --- db_transfer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index a422aa11..c09bcfdb 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -22,6 +22,7 @@ def __init__(self): self.user_pass = {} self.port_uid_table = {} self.onlineuser_cache = lru_cache.LRUCache(timeout=60*30) + self.start_time = time.time() def update_all_user(self, dt_transfer): import cymysql @@ -362,7 +363,7 @@ def load(self): return os.popen("cat /proc/loadavg | awk '{ print $1\" \"$2\" \"$3 }'").readlines()[0] def uptime(self): - return time.clock() + return time.time() - self.start_time def traffic_format(self, traffic): if traffic < 1024 * 8: From 0a7d71a39358a046369eea22bef44c9df9990d69 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 21 Jul 2016 14:59:19 +0800 Subject: [PATCH 246/487] using mysql.json to config mysql --- apiconfig.py | 30 +---- db_transfer.py | 244 +++++++++++++++++++++--------------- mysql.json | 13 ++ shadowsocks/crypto/table.py | 7 ++ switchrule.py | 9 +- 5 files changed, 166 insertions(+), 137 deletions(-) create mode 100644 mysql.json diff --git a/apiconfig.py b/apiconfig.py index 6576e7e5..58ea33a2 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,35 +1,15 @@ # Config -TRANSFER_MUL = 1.0 -NODE_ID = 1 -SERVER_PUB_ADDR = '127.0.0.1' # mujson_mgr need this to generate ssr link API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) +UPDATE_TIME = 60 +SERVER_PUB_ADDR = '127.0.0.1' # mujson_mgr need this to generate ssr link #mudb MUDB_FILE = 'mudb.json' # Mysql -MYSQL_HOST = '127.0.0.1' -MYSQL_PORT = 3306 -MYSQL_USER = 'ss' -MYSQL_PASS = 'ss' -MYSQL_DB = 'shadowsocks' -MYSQL_UPDATE_TIME = 60 - -MYSQL_SSL_ENABLE = 0 -MYSQL_SSL_CA = '' -MYSQL_SSL_CERT = '' -MYSQL_SSL_KEY = '' +MYSQL_CONFIG = 'usermysql.json' # API -API_HOST = '127.0.0.1' -API_PORT = 80 -API_PATH = '/mu/v2/' -API_TOKEN = 'abcdef' -API_UPDATE_TIME = 60 +MUAPI_CONFIG = 'usermuapi.json' + -# Manager (ignore this) -MANAGE_PASS = 'ss233333333' -#if you want manage in other server you should set this value to global ip -MANAGE_BIND_IP = '127.0.0.1' -#make sure this port is idle -MANAGE_PORT = 23333 diff --git a/db_transfer.py b/db_transfer.py index c09bcfdb..57811588 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -13,65 +13,19 @@ switchrule = None db_instance = None -class DbTransfer(object): +class TransferBase(object): def __init__(self): import threading + self.event = threading.Event() + self.key_list = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable'] self.last_get_transfer = {} self.last_update_transfer = {} - self.event = threading.Event() self.user_pass = {} self.port_uid_table = {} self.onlineuser_cache = lru_cache.LRUCache(timeout=60*30) - self.start_time = time.time() - - def update_all_user(self, dt_transfer): - import cymysql - update_transfer = {} - - query_head = 'UPDATE user' - query_sub_when = '' - query_sub_when2 = '' - query_sub_in = None - last_time = time.time() - for id in dt_transfer.keys(): - transfer = dt_transfer[id] - update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) - if transfer[0] + transfer[1] < update_trs: - continue - if id in self.user_pass: - del self.user_pass[id] - - query_sub_when += ' WHEN %s THEN u+%s' % (id, int(transfer[0] * get_config().TRANSFER_MUL)) - query_sub_when2 += ' WHEN %s THEN d+%s' % (id, int(transfer[1] * get_config().TRANSFER_MUL)) - update_transfer[id] = transfer - - if query_sub_in is not None: - query_sub_in += ',%s' % id - else: - query_sub_in = '%s' % id - - if query_sub_when == '': - return update_transfer - query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ - ' END, d = CASE port' + query_sub_when2 + \ - ' END, t = ' + str(int(last_time)) + \ - ' WHERE port IN (%s)' % query_sub_in - if get_config().MYSQL_SSL_ENABLE == 1: - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, - user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, - db=get_config().MYSQL_DB, charset='utf8', - ssl={'ca':get_config().MYSQL_SSL_CA,'cert':get_config().MYSQL_SSL_CERT,'key':get_config().MYSQL_SSL_KEY}) - else: - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, - user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, - db=get_config().MYSQL_DB, charset='utf8') - cur = conn.cursor() - cur.execute(query_sql) - cur.close() - conn.commit() - conn.close() - return update_transfer + def load_cfg(self): + pass def push_db_all_user(self): #更新用户流量到数据库 @@ -106,46 +60,9 @@ def push_db_all_user(self): self.last_update_transfer[id] = [last[0] + update_transfer[id][0], last[1] + update_transfer[id][1]] self.last_get_transfer = curr_transfer - def pull_db_all_user(self): - import cymysql - #数据库所有用户信息 - try: - switchrule = importloader.load('switchrule') - keys = switchrule.getKeys() - except Exception as e: - keys = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] - if get_config().MYSQL_SSL_ENABLE == 1: - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, - user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, - db=get_config().MYSQL_DB, charset='utf8', - ssl={'ca':get_config().MYSQL_SSL_CA,'cert':get_config().MYSQL_SSL_CERT,'key':get_config().MYSQL_SSL_KEY}) - else: - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, - user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, - db=get_config().MYSQL_DB, charset='utf8') - cur = conn.cursor() - cur.execute("SELECT " + ','.join(keys) + " FROM user") - rows = [] - for r in cur.fetchall(): - d = {} - for column in range(len(keys)): - d[keys[column]] = r[column] - rows.append(d) - cur.close() - conn.close() - return rows - - def cmp(self, val1, val2): - if type(val1) is bytes: - val1 = common.to_str(val1) - if type(val2) is bytes: - val2 = common.to_str(val2) - return val1 == val2 - def del_server_out_of_bound_safe(self, last_rows, rows): #停止超流量的服务 #启动没超流量的服务 - #需要动态载入switchrule，以便实时修改规则 try: switchrule = importloader.load('switchrule') except Exception as e: @@ -230,6 +147,13 @@ def del_server_out_of_bound_safe(self, last_rows, rows): logging.info('db start server at port [%s] pass [%s] protocol [%s] obfs [%s]' % (port, passwd, protocol, obfs)) ServerPool.get_instance().new_server(port, cfg) + def cmp(self, val1, val2): + if type(val1) is bytes: + val1 = common.to_str(val1) + if type(val2) is bytes: + val2 = common.to_str(val2) + return val1 == val2 + @staticmethod def del_servers(): for port in [v for v in ServerPool.get_instance().tcp_servers_pool.keys()]: @@ -251,6 +175,7 @@ def thread_db(obj): try: while True: load_config() + db_instance.load_cfg() try: db_instance.push_db_all_user() rows = db_instance.pull_db_all_user() @@ -260,7 +185,7 @@ def thread_db(obj): trace = traceback.format_exc() logging.error(trace) #logging.warn('db thread except:%s' % e) - if db_instance.event.wait(get_config().MYSQL_UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive(): + if db_instance.event.wait(get_config().UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive(): break except KeyboardInterrupt as e: pass @@ -273,9 +198,120 @@ def thread_db_stop(): global db_instance db_instance.event.set() +class DbTransfer(TransferBase): + def __init__(self): + super(DbTransfer, self).__init__() + self.cfg = { + "host": "127.0.0.1", + "port": 3306, + "user": "ss", + "password": "pass", + "db": "shadowsocks", + "node_id": 1, + "transfer_mul": 1.0, + "ssl_enable": 0, + "ssl_ca": "", + "ssl_cert": "", + "ssl_key": ""} + self.load_cfg() + + def load_cfg(self): + import json + config_path = get_config().MYSQL_CONFIG + cfg = None + with open(config_path, 'r+') as f: + cfg = json.loads(f.read().decode('utf8')) + + if cfg: + self.cfg.update(cfg) + + def update_all_user(self, dt_transfer): + import cymysql + update_transfer = {} + + query_head = 'UPDATE user' + query_sub_when = '' + query_sub_when2 = '' + query_sub_in = None + last_time = time.time() + + for id in dt_transfer.keys(): + transfer = dt_transfer[id] + update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) + if transfer[0] + transfer[1] < update_trs: + continue + if id in self.user_pass: + del self.user_pass[id] + + query_sub_when += ' WHEN %s THEN u+%s' % (id, int(transfer[0] * self.cfg["transfer_mul"])) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, int(transfer[1] * self.cfg["transfer_mul"])) + update_transfer[id] = transfer + + if query_sub_in is not None: + query_sub_in += ',%s' % id + else: + query_sub_in = '%s' % id + + if query_sub_when == '': + return update_transfer + query_sql = query_head + ' SET u = CASE port' + query_sub_when + \ + ' END, d = CASE port' + query_sub_when2 + \ + ' END, t = ' + str(int(last_time)) + \ + ' WHERE port IN (%s)' % query_sub_in + if self.cfg["ssl_enable"] == 1: + conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], + user=self.cfg["user"], passwd=self.cfg["password"], + db=self.cfg["db"], charset='utf8', + ssl={'ca':self.cfg["ssl_enable"],'cert':self.cfg["ssl_enable"],'key':self.cfg["ssl_enable"]}) + else: + conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], + user=self.cfg["user"], passwd=self.cfg["password"], + db=self.cfg["db"], charset='utf8') + + cur = conn.cursor() + cur.execute(query_sql) + cur.close() + conn.commit() + conn.close() + return update_transfer + + def pull_db_all_user(self): + import cymysql + #数据库所有用户信息 + try: + switchrule = importloader.load('switchrule') + keys = switchrule.getKeys(self.key_list) + except Exception as e: + keys = self.key_list + if self.cfg["ssl_enable"] == 1: + conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], + user=self.cfg["user"], passwd=self.cfg["password"], + db=self.cfg["db"], charset='utf8', + ssl={'ca':self.cfg["ssl_enable"],'cert':self.cfg["ssl_enable"],'key':self.cfg["ssl_enable"]}) + else: + conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], + user=self.cfg["user"], passwd=self.cfg["password"], + db=self.cfg["db"], charset='utf8') + + cur = conn.cursor() + cur.execute("SELECT " + ','.join(keys) + " FROM user") + rows = [] + for r in cur.fetchall(): + d = {} + for column in range(len(keys)): + d[keys[column]] = r[column] + rows.append(d) + cur.close() + conn.close() + return rows + class Dbv3Transfer(DbTransfer): def __init__(self): super(Dbv3Transfer, self).__init__() + self.key_list += ['id', 'method'] + if get_config().API_INTERFACE == 'sspanelv3ssr': + self.key_list += ['obfs', 'protocol', 'obfs_param', 'protocol_param'] + self.start_time = time.time() def update_all_user(self, dt_transfer): import cymysql @@ -290,15 +326,15 @@ def update_all_user(self, dt_transfer): alive_user_count = len(self.onlineuser_cache) bandwidth_thistime = 0 - if get_config().MYSQL_SSL_ENABLE == 1: - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, - user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, - db=get_config().MYSQL_DB, charset='utf8', - ssl={'ca':get_config().MYSQL_SSL_CA,'cert':get_config().MYSQL_SSL_CERT,'key':get_config().MYSQL_SSL_KEY}) + if self.cfg["ssl_enable"] == 1: + conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], + user=self.cfg["user"], passwd=self.cfg["password"], + db=self.cfg["db"], charset='utf8', + ssl={'ca':self.cfg["ssl_enable"],'cert':self.cfg["ssl_enable"],'key':self.cfg["ssl_enable"]}) else: - conn = cymysql.connect(host=get_config().MYSQL_HOST, port=get_config().MYSQL_PORT, - user=get_config().MYSQL_USER, passwd=get_config().MYSQL_PASS, - db=get_config().MYSQL_DB, charset='utf8') + conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], + user=self.cfg["user"], passwd=self.cfg["password"], + db=self.cfg["db"], charset='utf8') conn.autocommit(True) for id in dt_transfer.keys(): @@ -312,8 +348,8 @@ def update_all_user(self, dt_transfer): if id in self.user_pass: del self.user_pass[id] - query_sub_when += ' WHEN %s THEN u+%s' % (id, int(transfer[0] * get_config().TRANSFER_MUL)) - query_sub_when2 += ' WHEN %s THEN d+%s' % (id, int(transfer[1] * get_config().TRANSFER_MUL)) + query_sub_when += ' WHEN %s THEN u+%s' % (id, int(transfer[0] * self.cfg["transfer_mul"])) + query_sub_when2 += ' WHEN %s THEN d+%s' % (id, int(transfer[1] * self.cfg["transfer_mul"])) update_transfer[id] = transfer cur = conn.cursor() @@ -321,8 +357,8 @@ def update_all_user(self, dt_transfer): if id in self.port_uid_table: cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `node_id`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ - str(get_config().NODE_ID) + "', '" + str(get_config().TRANSFER_MUL) + "', '" + \ - self.traffic_format((transfer[0] + transfer[1]) * get_config().TRANSFER_MUL) + "', unix_timestamp()); ") + str(self.cfg["node_id"]) + "', '" + str(self.cfg["transfer_mul"]) + "', '" + \ + self.traffic_format((transfer[0] + transfer[1]) * self.cfg["transfer_mul"]) + "', unix_timestamp()); ") except: logging.warn('no `user_traffic_log` in db') cur.close() @@ -344,12 +380,12 @@ def update_all_user(self, dt_transfer): try: cur = conn.cursor() cur.execute("INSERT INTO `ss_node_online_log` (`id`, `node_id`, `online_user`, `log_time`) VALUES (NULL, '" + \ - str(get_config().NODE_ID) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") + str(self.cfg["node_id"]) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") cur.close() cur = conn.cursor() cur.execute("INSERT INTO `ss_node_info_log` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ - str(get_config().NODE_ID) + "', '" + str(self.uptime()) + "', '" + \ + str(self.cfg["node_id"]) + "', '" + str(self.uptime()) + "', '" + \ str(self.load()) + "', unix_timestamp()); ") cur.close() except: @@ -374,7 +410,7 @@ def traffic_format(self, traffic): return str(round((traffic / 1048576.0), 2)) + "MB"; -class MuJsonTransfer(DbTransfer): +class MuJsonTransfer(TransferBase): def __init__(self): super(MuJsonTransfer, self).__init__() diff --git a/mysql.json b/mysql.json new file mode 100644 index 00000000..3eb3bdad --- /dev/null +++ b/mysql.json @@ -0,0 +1,13 @@ +{ + "host": "127.0.0.1", + "port": 3306, + "user": "ss", + "password": "pass", + "db": "shadowsocks", + "node_id": 1, + "transfer_mul": 1.0, + "ssl_enable": 0, + "ssl_ca": "", + "ssl_cert": "", + "ssl_key": "" +} diff --git a/shadowsocks/crypto/table.py b/shadowsocks/crypto/table.py index bc693f51..c5d613f4 100644 --- a/shadowsocks/crypto/table.py +++ b/shadowsocks/crypto/table.py @@ -65,8 +65,15 @@ def update(self, data): else: return translate(data, self._decrypt_table) +class NoneCipher(object): + def __init__(self, cipher_name, key, iv, op): + pass + + def update(self, data): + return data ciphers = { + 'none': (0, 0, NoneCipher), 'table': (0, 0, TableCipher) } diff --git a/switchrule.py b/switchrule.py index 571a19f0..6687e12c 100644 --- a/switchrule.py +++ b/switchrule.py @@ -1,11 +1,4 @@ -from configloader import load_config, get_config - -def getKeys(): - key_list = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable' ] - if get_config().API_INTERFACE == 'sspanelv3': - key_list += ['id', 'method'] - elif get_config().API_INTERFACE == 'sspanelv3ssr': - key_list += ['id', 'method', 'obfs', 'protocol', 'obfs_param', 'protocol_param'] +def getKeys(key_list): return key_list #return key_list + ['plan'] # append the column name 'plan' From a6162496adcdf97186d66eb59cfafc5a292d3b92 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 21 Jul 2016 17:51:02 +0800 Subject: [PATCH 247/487] read traffic_rate from ss_node --- db_transfer.py | 52 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 57811588..fe6ece72 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -278,11 +278,6 @@ def update_all_user(self, dt_transfer): def pull_db_all_user(self): import cymysql #数据库所有用户信息 - try: - switchrule = importloader.load('switchrule') - keys = switchrule.getKeys(self.key_list) - except Exception as e: - keys = self.key_list if self.cfg["ssl_enable"] == 1: conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], user=self.cfg["user"], passwd=self.cfg["password"], @@ -293,6 +288,17 @@ def pull_db_all_user(self): user=self.cfg["user"], passwd=self.cfg["password"], db=self.cfg["db"], charset='utf8') + rows = self.pull_db_users(conn) + conn.close() + return rows + + def pull_db_users(self, conn): + try: + switchrule = importloader.load('switchrule') + keys = switchrule.getKeys(self.key_list) + except Exception as e: + keys = self.key_list + cur = conn.cursor() cur.execute("SELECT " + ','.join(keys) + " FROM user") rows = [] @@ -302,7 +308,6 @@ def pull_db_all_user(self): d[keys[column]] = r[column] rows.append(d) cur.close() - conn.close() return rows class Dbv3Transfer(DbTransfer): @@ -394,6 +399,39 @@ def update_all_user(self, dt_transfer): conn.close() return update_transfer + def pull_db_users(self, conn): + try: + switchrule = importloader.load('switchrule') + keys = switchrule.getKeys(self.key_list) + except Exception as e: + keys = self.key_list + + cur = conn.cursor() + + cur.execute("SELECT `traffic_rate` FROM ss_node where `id`='" + str(get_config().NODE_ID) + "'") + nodeinfo = cur.fetchone() + + if nodeinfo == None : + rows = [] + cur.close() + conn.commit() + conn.close() + return rows + cur.close() + + self.cfg['transfer_mul'] = float(nodeinfo['traffic_rate']) + + cur = conn.cursor() + cur.execute("SELECT " + ','.join(keys) + " FROM user") + rows = [] + for r in cur.fetchall(): + d = {} + for column in range(len(keys)): + d[keys[column]] = r[column] + rows.append(d) + cur.close() + return rows + def load(self): import os return os.popen("cat /proc/loadavg | awk '{ print $1\" \"$2\" \"$3 }'").readlines()[0] @@ -403,7 +441,7 @@ def uptime(self): def traffic_format(self, traffic): if traffic < 1024 * 8: - return str(traffic) + "B"; + return str(int(traffic)) + "B"; if traffic < 1024 * 1024 * 2: return str(round((traffic / 1024.0), 2)) + "KB"; From 0ed10c73c9359c48f88295dd4d298d14c056a108 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 24 Jul 2016 17:11:05 +0800 Subject: [PATCH 248/487] remove dup close --- db_transfer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index fe6ece72..5749e3e4 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -415,7 +415,6 @@ def pull_db_users(self, conn): rows = [] cur.close() conn.commit() - conn.close() return rows cur.close() From b922d70a87167174d438ab01e2ac15c48b5ea9dd Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 24 Jul 2016 17:39:16 +0800 Subject: [PATCH 249/487] set pull flag --- db_transfer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db_transfer.py b/db_transfer.py index 5749e3e4..a234ce61 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -23,11 +23,14 @@ def __init__(self): self.user_pass = {} self.port_uid_table = {} self.onlineuser_cache = lru_cache.LRUCache(timeout=60*30) + self.pull_ok = False def load_cfg(self): pass def push_db_all_user(self): + if self.pull_ok is False: + return #更新用户流量到数据库 last_transfer = self.last_update_transfer curr_transfer = ServerPool.get_instance().get_servers_transfer() @@ -179,6 +182,8 @@ def thread_db(obj): try: db_instance.push_db_all_user() rows = db_instance.pull_db_all_user() + if rows: + db_instance.pull_ok = True db_instance.del_server_out_of_bound_safe(last_rows, rows) last_rows = rows except Exception as e: From 2c4600cf0e60d9a26766d20c0e7ef6a850081272 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 24 Jul 2016 22:55:08 +0800 Subject: [PATCH 250/487] fix get node_id --- db_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index a234ce61..dbb04278 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -413,7 +413,7 @@ def pull_db_users(self, conn): cur = conn.cursor() - cur.execute("SELECT `traffic_rate` FROM ss_node where `id`='" + str(get_config().NODE_ID) + "'") + cur.execute("SELECT `traffic_rate` FROM ss_node where `id`='" + str(self.cfg["node_id"]) + "'") nodeinfo = cur.fetchone() if nodeinfo == None : From 3525d3d86aab16b7b5d096d5959df88c13554608 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 24 Jul 2016 23:06:54 +0800 Subject: [PATCH 251/487] fix get node_id --- db_transfer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index dbb04278..ae7a134c 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -413,17 +413,21 @@ def pull_db_users(self, conn): cur = conn.cursor() - cur.execute("SELECT `traffic_rate` FROM ss_node where `id`='" + str(self.cfg["node_id"]) + "'") + node_info_keys = ['traffic_rate'] + cur.execute("SELECT " + ','.join(node_info_keys) +" FROM ss_node where `id`='" + str(self.cfg["node_id"]) + "'") nodeinfo = cur.fetchone() - if nodeinfo == None : + if nodeinfo == None: rows = [] cur.close() conn.commit() return rows cur.close() - self.cfg['transfer_mul'] = float(nodeinfo['traffic_rate']) + node_info_dict = {} + for column in range(len(nodeinfo)): + node_info_dict[node_info_keys[column]] = r[column] + self.cfg['transfer_mul'] = float(node_info_dict['traffic_rate']) cur = conn.cursor() cur.execute("SELECT " + ','.join(keys) + " FROM user") From a440a726f6d3a25fe2eeac94786580f462a314a7 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sun, 24 Jul 2016 23:08:45 +0800 Subject: [PATCH 252/487] fix get node_id --- db_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index ae7a134c..b5abd55d 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -426,7 +426,7 @@ def pull_db_users(self, conn): node_info_dict = {} for column in range(len(nodeinfo)): - node_info_dict[node_info_keys[column]] = r[column] + node_info_dict[node_info_keys[column]] = nodeinfo[column] self.cfg['transfer_mul'] = float(node_info_dict['traffic_rate']) cur = conn.cursor() From 0213e7884755dd8ef28600ab921427fc5ecda223 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 27 Jul 2016 14:23:48 +0800 Subject: [PATCH 253/487] log listening port, remove `auth_simple`, redirect the same port --- shadowsocks/obfsplugin/auth.py | 164 +-------------------------------- shadowsocks/tcprelay.py | 45 +++++---- 2 files changed, 31 insertions(+), 178 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 012236ff..afe27bce 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -36,9 +36,6 @@ from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord, chr -def create_auth_simple(method): - return auth_simple(method) - def create_auth_sha1(method): return auth_sha1(method) @@ -46,7 +43,6 @@ def create_auth_sha1_v2(method): return auth_sha1_v2(method) obfs_map = { - 'auth_simple': (create_auth_simple,), 'auth_sha1': (create_auth_sha1,), 'auth_sha1_compatible': (create_auth_sha1,), 'auth_sha1_v2': (create_auth_sha1_v2,), @@ -137,7 +133,7 @@ def __init__(self): self.startup_time = int(time.time() - 30) & 0xFFFFFFFF self.local_client_id = b'' self.connection_id = 0 - self.set_max_client(16) # max active client count + self.set_max_client(64) # max active client count def update(self, client_id, connection_id): if client_id in self.client_id: @@ -181,162 +177,6 @@ def insert(self, client_id, connection_id): else: return self.client_id[client_id].insert(connection_id) -class auth_simple(verify_base): - def __init__(self, method): - super(auth_simple, self).__init__(method) - self.recv_buf = b'' - self.unit_len = 8100 - self.decrypt_packet_num = 0 - self.raw_trans = False - self.has_sent_header = False - self.has_recv_header = False - self.client_id = 0 - self.connection_id = 0 - self.max_time_dif = 60 * 5 # time dif (second) setting - - def init_data(self): - return obfs_auth_data() - - def set_server_info(self, server_info): - self.server_info = server_info - try: - max_client = int(server_info.protocol_param) - except: - max_client = 16 - self.server_info.data.set_max_client(max_client) - - def pack_data(self, buf): - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) - data = common.chr(len(rnd_data) + 1) + rnd_data + buf - data = struct.pack('>H', len(data) + 6) + data - crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff - data += struct.pack(' 0xFF000000: - self.server_info.data.local_client_id = b'' - if not self.server_info.data.local_client_id: - self.server_info.data.local_client_id = os.urandom(4) - logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) - self.server_info.data.connection_id = struct.unpack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def client_post_decrypt(self, buf): - if self.raw_trans: - return buf - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data error') - if length > len(self.recv_buf): - break - - if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data uncorrect CRC32') - - pos = common.ord(self.recv_buf[2]) + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - - if out_buf: - self.decrypt_packet_num += 1 - return out_buf - - def server_pre_encrypt(self, buf): - ret = b'' - while len(buf) > self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def server_post_decrypt(self, buf): - if self.raw_trans: - return (buf, False) - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - logging.info('auth_simple: over size') - return (b'E', False) - else: - raise Exception('server_post_decrype data error') - if length > len(self.recv_buf): - break - - if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: - logging.info('auth_simple: crc32 error, data %s' % (binascii.hexlify(self.recv_buf[:length]),)) - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return (b'E', False) - else: - raise Exception('server_post_decrype data uncorrect CRC32') - - pos = common.ord(self.recv_buf[2]) + 2 - out_buf += self.recv_buf[pos:length - 4] - if not self.has_recv_header: - if len(out_buf) < 12: - self.raw_trans = True - self.recv_buf = b'' - logging.info('auth_simple: too short') - return (b'E', False) - utc_time = struct.unpack(' self.max_time_dif \ - or common.int32(utc_time - self.server_info.data.startup_time) < 0: - self.raw_trans = True - self.recv_buf = b'' - logging.info('auth_simple: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) - return (b'E', False) - elif self.server_info.data.insert(client_id, connection_id): - self.has_recv_header = True - out_buf = out_buf[12:] - self.client_id = client_id - self.connection_id = connection_id - else: - self.raw_trans = True - self.recv_buf = b'' - logging.info('auth_simple: auth fail, data %s' % (binascii.hexlify(out_buf),)) - return (b'E', False) - self.recv_buf = self.recv_buf[length:] - - if out_buf: - self.server_info.data.update(self.client_id, self.connection_id) - self.decrypt_packet_num += 1 - return (out_buf, False) - class auth_sha1(verify_base): def __init__(self, method): super(auth_sha1, self).__init__(method) @@ -476,7 +316,7 @@ def server_post_decrypt(self, buf): utc_time = struct.unpack(' self.max_time_dif \ or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: self.raw_trans = True diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 085565cb..34a0eaf5 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -345,23 +345,37 @@ def _get_redirect_host(self, client_address, ogn_data): addrs = socket.getaddrinfo(client_address[0], client_address[1], 0, socket.SOCK_STREAM, socket.SOL_TCP) af, socktype, proto, canonname, sa = addrs[0] address_bytes = common.inet_pton(af, sa[0]) - if len(address_bytes) == 16: + if af == socket.AF_INET6: addr = struct.unpack('>Q', address_bytes[8:])[0] - if len(address_bytes) == 4: + elif af == socket.AF_INET: addr = struct.unpack('>I', address_bytes)[0] else: addr = 0 + + host_port = [] + match_port = False if type(host_list) == list: - host_post = common.to_str(host_list[((hash_code & 0xffffffff) + addr) % len(host_list)]) - else: - host_post = common.to_str(host_list) - items = host_post.rsplit(':', 1) - if len(items) > 1: - try: - return (items[0], int(items[1])) - except: - pass - return (host_post, 80) + for host in host_list: + items = common.to_str(host).rsplit(':', 1) + if len(items) > 1: + try: + port = int(items[1]) + if port == self._server._listen_port: + match_port = True + host_port.append((items[0], port)) + except: + pass + else: + host_port.append((host, 80)) + + if match_port: + last_host_port = host_port + host_port = [] + for host in last_host_port: + if host[1] == self._server._listen_port: + host_port.append(host) + + return host_port[((hash_code & 0xffffffff) + addr) % len(host_port)] def _handel_protocol_error(self, client_address, ogn_data): logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d via port %d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1], self._server._listen_port)) @@ -473,10 +487,9 @@ def _handle_stage_addr(self, ogn_data, data): data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) connecttype, remote_addr, remote_port, header_length = header_result - common.connect_log('%s connecting %s:%d from %s:%d' % + common.connect_log('%s connecting %s:%d via port %d' % ((connecttype == 0) and 'TCP' or 'UDP', - common.to_str(remote_addr), remote_port, - self._client_address[0], self._client_address[1])) + common.to_str(remote_addr), remote_port, self._server._listen_port)) self._remote_address = (common.to_str(remote_addr), remote_port) self._remote_udp = (connecttype != 0) # pause reading @@ -815,7 +828,7 @@ def _on_remote_error(self): if self._remote_sock: logging.error(eventloop.get_sock_error(self._remote_sock)) if self._remote_address: - logging.error("when connect to %s:%d from %s:%d" % (self._remote_address[0], self._remote_address[1], self._client_address[0], self._client_address[1])) + logging.error("when connect to %s:%d" % (self._remote_address[0], self._remote_address[1])) else: logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() From f0d6a5b41ee1f11016166f6d868c8b0a947efc75 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Wed, 27 Jul 2016 18:09:57 +0800 Subject: [PATCH 254/487] log listening port over UDP --- shadowsocks/udprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index fff62c10..6f595277 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1111,9 +1111,9 @@ def _handle_server(self): logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) - common.connect_log('UDP data to %s:%d from %s:%d' % + common.connect_log('UDP data to %s:%d via port %d' % (common.to_str(server_addr), server_port, - r_addr[0], r_addr[1])) + self._listen_port)) self._cache.clear(self._udp_cache_size) self._cache_dns_client.clear(16) From 1121434a5f8fda02c03c09c9b9afd6125531c9fe Mon Sep 17 00:00:00 2001 From: chenss Date: Thu, 28 Jul 2016 21:21:10 +0800 Subject: [PATCH 255/487] add shadowsocks.obfsplugin to setup.py for distribution --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 689dd736..50a38081 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ author='clowwindy', author_email='clowwindy42@gmail.com', url='https://github.com/shadowsocks/shadowsocks', - packages=['shadowsocks', 'shadowsocks.crypto'], + packages=['shadowsocks', 'shadowsocks.crypto', 'shadowsocks.obfsplugin'], package_data={ 'shadowsocks': ['README.rst', 'LICENSE'] }, From 82df4524d18ae04b46fe695cf657d3f960d9f037 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Thu, 28 Jul 2016 23:18:18 +0800 Subject: [PATCH 256/487] fix host_list type --- shadowsocks/tcprelay.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 34a0eaf5..ef9a1c13 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -354,19 +354,20 @@ def _get_redirect_host(self, client_address, ogn_data): host_port = [] match_port = False - if type(host_list) == list: - for host in host_list: - items = common.to_str(host).rsplit(':', 1) - if len(items) > 1: - try: - port = int(items[1]) - if port == self._server._listen_port: - match_port = True - host_port.append((items[0], port)) - except: - pass - else: - host_port.append((host, 80)) + if type(host_list) != list: + host_list = [host_list] + for host in host_list: + items = common.to_str(host).rsplit(':', 1) + if len(items) > 1: + try: + port = int(items[1]) + if port == self._server._listen_port: + match_port = True + host_port.append((items[0], port)) + except: + pass + else: + host_port.append((host, 80)) if match_port: last_host_port = host_port From 4a291ff335ada38d061c1e1eb7b20e192dece6d4 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 29 Jul 2016 17:19:30 +0800 Subject: [PATCH 257/487] add keep alive setting --- shadowsocks/tcprelay.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index ef9a1c13..e70de022 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -563,6 +563,11 @@ def _create_remote_socket(self, ip, port): remote_sock_v6.setblocking(False) else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 20) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5) + if not self._is_local: bind_addr = '' if self._bind and af == socket.AF_INET: From 25e9c131d4aecf19528e0e8dba4beb3df307ce74 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Fri, 29 Jul 2016 21:38:05 +0800 Subject: [PATCH 258/487] change err return --- shadowsocks/obfsplugin/http_simple.py | 6 +++--- shadowsocks/obfsplugin/obfs_tls.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 6783a32e..6937213e 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -145,7 +145,7 @@ def not_match_return(self, buf): self.has_sent_header = True self.has_recv_header = True if self.method == 'http_simple': - return (b'E', False, False) + return (b'E'*64, False, False) return (buf, True, False) def server_decode(self, buf): @@ -212,7 +212,7 @@ def not_match_return(self, buf): self.has_sent_header = True self.has_recv_header = True if self.method == 'http_post': - return (b'E', False, False) + return (b'E'*64, False, False) return (buf, True, False) def server_decode(self, buf): @@ -292,7 +292,7 @@ def server_decode(self, buf): if crc != 0xffffffff: self.has_sent_header = True if self.method == 'random_head': - return (b'E', False, False) + return (b'E'*64, False, False) return (buf, True, False) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 529444ee..e217d12d 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -104,7 +104,7 @@ def server_encode(self, buf): def decode_error_return(self, buf): self.has_sent_header = True if self.method == 'tls_simple': - return (b'E', False, False) + return (b'E'*64, False, False) return (buf, True, False) def server_decode(self, buf): @@ -205,7 +205,7 @@ def decode_error_return(self, buf): self.raw_trans_sent = True self.raw_trans_recv = True if self.method == 'tls1.0_session_auth': - return (b'E', False, False) + return (b'E'*64, False, False) return (buf, True, False) def server_decode(self, buf): @@ -384,7 +384,7 @@ def server_encode(self, buf): def decode_error_return(self, buf): self.handshake_status = -1 if self.method == 'tls1.2_session_auth': - return (b'E', False, False) + return (b'E'*64, False, False) return (buf, True, False) def server_decode(self, buf): From 8da3bdc5117d6fbd2e75f130fbfd2174813a2ee0 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 30 Jul 2016 09:56:41 +0800 Subject: [PATCH 259/487] set KEEPALIVE except windows --- shadowsocks/tcprelay.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e70de022..968e4d10 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -26,6 +26,7 @@ import binascii import traceback import random +import platform from shadowsocks import encrypt, obfs, eventloop, shell, common from shadowsocks.common import pre_parse_header, parse_header @@ -564,9 +565,10 @@ def _create_remote_socket(self, ip, port): else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 20) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5) + if platform.system() != 'Windows': + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 20) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5) if not self._is_local: bind_addr = '' From f14461ea648dd5393785e64dd2172b2050b6110a Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 30 Jul 2016 09:58:01 +0800 Subject: [PATCH 260/487] replace("=", "") --- mujson_mgr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index a9cf0f5f..af7befc4 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -59,8 +59,8 @@ def ssrlink(self, user, encode): obfs = user.get('obfs', '') protocol = protocol.replace("_compatible", "") obfs = obfs.replace("_compatible", "") - link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd'])))) - return "ssr://" + ( encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))) or link) + link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace("=", "")) + return "ssr://" + ( encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) def userinfo(self, user): ret = "" From 292865f62556fad954b421cc8ad0b1c9e35d3d0c Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 30 Jul 2016 11:45:47 +0800 Subject: [PATCH 261/487] Linux only --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 968e4d10..a167e436 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -565,7 +565,7 @@ def _create_remote_socket(self, ip, port): else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - if platform.system() != 'Windows': + if platform.system() in ['Linux']: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120) remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 20) remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5) From e15030cc6e952225b3388915e90950788784c481 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 1 Aug 2016 13:19:24 +0800 Subject: [PATCH 262/487] set KEEPIDLE on windows/mac osx --- shadowsocks/tcprelay.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a167e436..e118a4b2 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -565,10 +565,21 @@ def _create_remote_socket(self, ip, port): else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + if self._is_local: + val_idle = 60 + val_intvl = 5 + else: + val_idle = 120 + val_intvl = 20 if platform.system() in ['Linux']: - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 20) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, val_idle) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, val_intvl) remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5) + elif platform.system() in ['Windows']: + remote_sock.ioctl(socket.SIO_KEEPALIVE_VALS, (1, 1000 * val_idle, 1000 * val_intvl)) + elif platform.system() in ["Darwin"]: #OSX + TCP_KEEPALIVE = 0x10 + sock.setsockopt(socket.SOL_TCP, TCP_KEEPALIVE, val_intvl * 2) if not self._is_local: bind_addr = '' From f3808a306fbafb1e2a451f1ac0b44e53666969a8 Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 1 Aug 2016 21:42:43 +0800 Subject: [PATCH 263/487] random choice a host name (http_simple/http_post/tls1.2_ticket_auth) --- shadowsocks/obfsplugin/http_simple.py | 38 +++++++++++++++++++++------ shadowsocks/obfsplugin/obfs_tls.py | 2 ++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 6937213e..103ce164 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -96,10 +96,21 @@ def client_encode(self, buf): port = b'' if self.server_info.port != 80: port = b':' + to_bytes(str(self.server_info.port)) + body = None + hosts = (self.server_info.obfs_param or self.server_info.host) + pos = hosts.find("#") + if pos >= 0: + body = hosts[pos + 1:].replace("\\n", "\r\n") + hosts = hosts[:pos] + hosts = hosts.split(',') + host = random.choice(hosts) http_head = b"GET /" + self.encode_head(headdata) + b" HTTP/1.1\r\n" - http_head += b"Host: " + to_bytes(self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" - http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" - http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\nDNT: 1\r\nConnection: keep-alive\r\n\r\n" + http_head += b"Host: " + to_bytes(host) + port + b"\r\n" + if body: + http_head += body + "\r\n\r\n" + else: + http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" + http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\nDNT: 1\r\nConnection: keep-alive\r\n\r\n" self.has_sent_header = True return http_head + buf @@ -199,12 +210,23 @@ def client_encode(self, buf): port = b'' if self.server_info.port != 80: port = b':' + to_bytes(str(self.server_info.port)) + body = None + hosts = (self.server_info.obfs_param or self.server_info.host) + pos = hosts.find("#") + if pos >= 0: + body = hosts[pos + 1:].replace("\\n", "\r\n") + hosts = hosts[:pos] + hosts = hosts.split(',') + host = random.choice(hosts) http_head = b"POST /" + self.encode_head(headdata) + b" HTTP/1.1\r\n" - http_head += b"Host: " + to_bytes(self.server_info.obfs_param or self.server_info.host) + port + b"\r\n" - http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" - http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\n" - http_head += b"Content-Type: multipart/form-data; boundary=" + self.boundary() + b"\r\nDNT: 1\r\n" - http_head += "Connection: keep-alive\r\n\r\n" + http_head += b"Host: " + to_bytes(host) + port + b"\r\n" + if body: + http_head += body + "\r\n\r\n" + else: + http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" + http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\n" + http_head += b"Content-Type: multipart/form-data; boundary=" + self.boundary() + b"\r\nDNT: 1\r\n" + http_head += "Connection: keep-alive\r\n\r\n" self.has_sent_header = True return http_head + buf diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index e217d12d..98c25c2c 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -312,6 +312,8 @@ def client_encode(self, buf): host = self.server_info.obfs_param or self.server_info.host if host and host[-1] in string.digits: host = '' + hosts = host.split(',') + host = random.choice(hosts) ext += self.sni(host) ext += b"\x00\x17\x00\x00" ext += b"\x00\x23\x00\xd0" + os.urandom(208) # ticket From d47905f01d1dc93dd4497439c9503ba2e3d30d3e Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Mon, 1 Aug 2016 22:56:37 +0800 Subject: [PATCH 264/487] fix name --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e118a4b2..a4e5c7b9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -579,7 +579,7 @@ def _create_remote_socket(self, ip, port): remote_sock.ioctl(socket.SIO_KEEPALIVE_VALS, (1, 1000 * val_idle, 1000 * val_intvl)) elif platform.system() in ["Darwin"]: #OSX TCP_KEEPALIVE = 0x10 - sock.setsockopt(socket.SOL_TCP, TCP_KEEPALIVE, val_intvl * 2) + remote_sock.setsockopt(socket.SOL_TCP, TCP_KEEPALIVE, val_intvl * 2) if not self._is_local: bind_addr = '' From b1aade864087c67e9c84a11f24e9a0d6bebb90cf Mon Sep 17 00:00:00 2001 From: BreakWa11 Date: Sat, 6 Aug 2016 18:26:56 +0800 Subject: [PATCH 265/487] remove custom obfs_param & protocol_param --- db_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index b5abd55d..a34cd76e 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -320,7 +320,7 @@ def __init__(self): super(Dbv3Transfer, self).__init__() self.key_list += ['id', 'method'] if get_config().API_INTERFACE == 'sspanelv3ssr': - self.key_list += ['obfs', 'protocol', 'obfs_param', 'protocol_param'] + self.key_list += ['obfs', 'protocol'] self.start_time = time.time() def update_all_user(self, dt_transfer): From 3ce6e6f71432997e27838f3791b4bf33cf2a98ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 13 Aug 2016 23:36:00 +0800 Subject: [PATCH 266/487] fix transfer update --- db_transfer.py | 64 +++++++++++++++++---------- shadowsocks/eventloop.py | 2 +- shadowsocks/obfsplugin/http_simple.py | 3 +- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index a34cd76e..d08a4422 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -18,12 +18,12 @@ def __init__(self): import threading self.event = threading.Event() self.key_list = ['port', 'u', 'd', 'transfer_enable', 'passwd', 'enable'] - self.last_get_transfer = {} - self.last_update_transfer = {} - self.user_pass = {} - self.port_uid_table = {} - self.onlineuser_cache = lru_cache.LRUCache(timeout=60*30) - self.pull_ok = False + self.last_get_transfer = {} #上一次的实际流量 + self.last_update_transfer = {} #上一次更新到的流量（小于等于实际流量） + self.force_update_transfer = set() #强制推入数据库的ID + self.port_uid_table = {} #端口到uid的映射（仅v3以上有用） + self.onlineuser_cache = lru_cache.LRUCache(timeout=60*30) #用户在线状态记录 + self.pull_ok = False #记录是否已经拉出过数据 def load_cfg(self): pass @@ -36,7 +36,19 @@ def push_db_all_user(self): curr_transfer = ServerPool.get_instance().get_servers_transfer() #上次和本次的增量 dt_transfer = {} + for id in self.force_update_transfer: #此表中的用户统计上次未计入的流量 + if id in self.last_get_transfer and id in last_transfer: + dt_transfer[id] = [self.last_get_transfer[id][0] - last_transfer[id][0], self.last_get_transfer[id][1] - last_transfer[id][1]] + for id in curr_transfer.keys(): + #有流量的，先记录在线状态 + if id in self.last_get_transfer: + if curr_transfer[id][0] + curr_transfer[id][1] > self.last_get_transfer[id][0] + self.last_get_transfer[id][1]: + self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] + else: + self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] + + #算出与上次记录的流量差值，保存于dt_transfer表 if id in last_transfer: if curr_transfer[id][0] + curr_transfer[id][1] - last_transfer[id][0] - last_transfer[id][1] <= 0: continue @@ -50,17 +62,18 @@ def push_db_all_user(self): if curr_transfer[id][0] + curr_transfer[id][1] <= 0: continue dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] - if id in self.last_get_transfer: - if curr_transfer[id][0] + curr_transfer[id][1] > self.last_get_transfer[id][0] + self.last_get_transfer[id][1]: - self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] - else: - self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] + self.onlineuser_cache.sweep() - update_transfer = self.update_all_user(dt_transfer) - for id in update_transfer.keys(): - last = self.last_update_transfer.get(id, [0,0]) - self.last_update_transfer[id] = [last[0] + update_transfer[id][0], last[1] + update_transfer[id][1]] + update_transfer = self.update_all_user(dt_transfer) #返回有更新的表 + for id in update_transfer.keys(): #其增量加在此表 + if id in self.force_update_transfer: #但排除在force_update_transfer内的 + if id in self.last_update_transfer: + del self.last_update_transfer[id] + self.force_update_transfer.remove(id) + else: + last = self.last_update_transfer.get(id, [0,0]) + self.last_update_transfer[id] = [last[0] + update_transfer[id][0], last[1] + update_transfer[id][1]] self.last_get_transfer = curr_transfer def del_server_out_of_bound_safe(self, last_rows, rows): @@ -125,11 +138,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): new_servers[port] = (passwd, cfg) elif allow and ServerPool.get_instance().server_run_status(port) is False: - #new_servers[port] = passwd - protocol = cfg.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) - obfs = cfg.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) - logging.info('db start server at port [%s] pass [%s] protocol [%s] obfs [%s]' % (port, passwd, protocol, obfs)) - ServerPool.get_instance().new_server(port, cfg) + self.new_server(port, passwd, cfg) for row in last_rows: if row['port'] in cur_servers: @@ -145,10 +154,15 @@ def del_server_out_of_bound_safe(self, last_rows, rows): self.event.wait(eventloop.TIMEOUT_PRECISION + eventloop.TIMEOUT_PRECISION / 2) for port in new_servers.keys(): passwd, cfg = new_servers[port] - protocol = cfg.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) - obfs = cfg.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) - logging.info('db start server at port [%s] pass [%s] protocol [%s] obfs [%s]' % (port, passwd, protocol, obfs)) - ServerPool.get_instance().new_server(port, cfg) + self.new_server(port, passwd, cfg) + + def new_server(self, port, passwd, cfg): + protocol = cfg.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) + method = cfg.get('method', ServerPool.get_instance().config.get('method', 'None')) + obfs = cfg.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) + logging.info('db start server at port [%s] pass [%s] protocol [%s] method [%s] obfs [%s]' % (port, passwd, protocol, method, obfs)) + ServerPool.get_instance().new_server(port, cfg) + self.force_update_transfer.add(port) def cmp(self, val1, val2): if type(val1) is bytes: @@ -206,6 +220,7 @@ def thread_db_stop(): class DbTransfer(TransferBase): def __init__(self): super(DbTransfer, self).__init__() + self.user_pass = {} #记录更新此用户流量时被跳过多少次 self.cfg = { "host": "127.0.0.1", "port": 3306, @@ -242,6 +257,7 @@ def update_all_user(self, dt_transfer): for id in dt_transfer.keys(): transfer = dt_transfer[id] + #小于最低更新流量的先不更新 update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) if transfer[0] + transfer[1] < update_trs: continue diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index ce9c11bc..2d7e696e 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -53,7 +53,7 @@ } # we check timeouts every TIMEOUT_PRECISION seconds -TIMEOUT_PRECISION = 10 +TIMEOUT_PRECISION = 5 class KqueueLoop(object): diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 103ce164..42715fcf 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -100,7 +100,8 @@ def client_encode(self, buf): hosts = (self.server_info.obfs_param or self.server_info.host) pos = hosts.find("#") if pos >= 0: - body = hosts[pos + 1:].replace("\\n", "\r\n") + body = hosts[pos + 1:].replace("\n", "\r\n") + body = body.replace("\\n", "\r\n") hosts = hosts[:pos] hosts = hosts.split(',') host = random.choice(hosts) From 816d2f629872eaf800b375f079710a3adb6a657f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 15 Aug 2016 12:49:52 +0800 Subject: [PATCH 267/487] remove tls_simple, fix tls1.2_ticket_auth --- shadowsocks/obfsplugin/obfs_tls.py | 74 +----------------------------- 1 file changed, 1 insertion(+), 73 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 98c25c2c..400e1aaa 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -35,9 +35,6 @@ from shadowsocks.common import to_bytes, to_str, ord from shadowsocks import lru_cache -def create_tls_obfs(method): - return tls_simple(method) - def create_tls_auth_obfs(method): return tls_auth(method) @@ -45,8 +42,6 @@ def create_tls_ticket_auth_obfs(method): return tls_ticket_auth(method) obfs_map = { - 'tls_simple': (create_tls_obfs,), - 'tls_simple_compatible': (create_tls_obfs,), 'tls1.0_session_auth': (create_tls_auth_obfs,), 'tls1.0_session_auth_compatible': (create_tls_auth_obfs,), 'tls1.2_ticket_auth': (create_tls_ticket_auth_obfs,), @@ -59,73 +54,6 @@ def match_begin(str1, str2): return True return False -class tls_simple(plain.plain): - def __init__(self, method): - self.method = method - self.has_sent_header = False - self.has_recv_header = False - self.raw_trans_sent = False - self.send_buffer = b'' - self.tls_version = b'\x03\x01' - - def client_encode(self, buf): - if self.raw_trans_sent: - return buf - self.send_buffer += buf - if not self.has_sent_header: - self.has_sent_header = True - data = self.tls_version + os.urandom(32) + binascii.unhexlify(b"000016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") - data = b"\x01\x00" + struct.pack('>H', len(data)) + data - data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data - return data - if self.has_recv_header: - ret = self.send_buffer - self.send_buffer = b'' - self.raw_trans_sent = True - return ret - return b'' - - def client_decode(self, buf): - if self.has_recv_header: - return (buf, False) - self.has_recv_header = True - return (b'', True) - - def server_encode(self, buf): - if self.has_sent_header: - return buf - self.has_sent_header = True - # TODO - data = self.tls_version + os.urandom(32) - data = b"\x02\x00" + struct.pack('>H', len(data)) + data - data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data - return data - - def decode_error_return(self, buf): - self.has_sent_header = True - if self.method == 'tls_simple': - return (b'E'*64, False, False) - return (buf, True, False) - - def server_decode(self, buf): - if self.has_recv_header: - return (buf, True, False) - - self.has_recv_header = True - if not match_begin(buf, b'\x16' + self.tls_version): - return self.decode_error_return(buf) - buf = buf[3:] - if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: - return self.decode_error_return(buf) - buf = buf[2:] - if not match_begin(buf, b'\x01\x00'): #client hello - return self.decode_error_return(buf) - buf = buf[2:] - if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: - return self.decode_error_return(buf) - # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) - return (b'', False, True) - class obfs_client_data(object): def __init__(self, cid): self.client_id = cid @@ -385,7 +313,7 @@ def server_encode(self, buf): def decode_error_return(self, buf): self.handshake_status = -1 - if self.method == 'tls1.2_session_auth': + if self.method == 'tls1.2_ticket_auth': return (b'E'*64, False, False) return (buf, True, False) From a26019f9e6843f9df08c8f84d46042c528580fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 15 Aug 2016 14:24:28 +0800 Subject: [PATCH 268/487] fix online users --- db_transfer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index d08a4422..a9657b89 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -41,13 +41,6 @@ def push_db_all_user(self): dt_transfer[id] = [self.last_get_transfer[id][0] - last_transfer[id][0], self.last_get_transfer[id][1] - last_transfer[id][1]] for id in curr_transfer.keys(): - #有流量的，先记录在线状态 - if id in self.last_get_transfer: - if curr_transfer[id][0] + curr_transfer[id][1] > self.last_get_transfer[id][0] + self.last_get_transfer[id][1]: - self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] - else: - self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] - #算出与上次记录的流量差值，保存于dt_transfer表 if id in last_transfer: if curr_transfer[id][0] + curr_transfer[id][1] - last_transfer[id][0] - last_transfer[id][1] <= 0: @@ -63,6 +56,13 @@ def push_db_all_user(self): continue dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + #有流量的，先记录在线状态 + if id in self.last_get_transfer: + if curr_transfer[id][0] + curr_transfer[id][1] > self.last_get_transfer[id][0] + self.last_get_transfer[id][1]: + self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] + else: + self.onlineuser_cache[id] = curr_transfer[id][0] + curr_transfer[id][1] + self.onlineuser_cache.sweep() update_transfer = self.update_all_user(dt_transfer) #返回有更新的表 From f989fd4d37dbec66ab847f59283e1cd2ce64018d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 16 Aug 2016 14:02:11 +0800 Subject: [PATCH 269/487] fix error return --- shadowsocks/obfsplugin/auth.py | 44 ++++++++++++++++------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index afe27bce..8b673f37 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -63,6 +63,7 @@ class verify_base(plain.plain): def __init__(self, method): super(verify_base, self).__init__(method) self.method = method + self.no_compatible_method = '' def init_data(self): return '' @@ -82,6 +83,12 @@ def server_encode(self, buf): def server_decode(self, buf): return (buf, True, False) + def not_match_return(self, buf): + self.raw_trans = True + if self.method == self.no_compatible_method: + return (b'E'*64, False) + return (buf, False) + class client_queue(object): def __init__(self, begin_id): self.front = begin_id - 64 @@ -189,6 +196,7 @@ def __init__(self, method): self.client_id = 0 self.connection_id = 0 self.max_time_dif = 60 * 60 # time dif (second) setting + self.no_compatible_method = 'auth_sha1' def init_data(self): return obfs_auth_data() @@ -294,45 +302,38 @@ def server_post_decrypt(self, buf): return (b'', False) crc = struct.pack('H', self.recv_buf[4:6])[0] if length > len(self.recv_buf): return (b'', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] if sha1data != self.recv_buf[length - 10:length]: logging.error('auth_sha1 data uncorrect auth HMAC-SHA1') - return (b'E', False) + return self.not_match_return(self.recv_buf) pos = common.ord(self.recv_buf[6]) + 6 out_buf = self.recv_buf[pos:length - 10] if len(out_buf) < 12: - self.raw_trans = True self.recv_buf = b'' logging.info('auth_sha1: too short') - return (b'E', False) + return self.not_match_return(self.recv_buf) utc_time = struct.unpack(' self.max_time_dif \ or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: - self.raw_trans = True self.recv_buf = b'' logging.info('auth_sha1: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) - return (b'E', False) + return self.not_match_return(self.recv_buf) elif self.server_info.data.insert(client_id, connection_id): self.has_recv_header = True out_buf = out_buf[12:] self.client_id = client_id self.connection_id = connection_id else: - self.raw_trans = True self.recv_buf = b'' logging.info('auth_sha1: auth fail, data %s' % (binascii.hexlify(out_buf),)) - return (b'E', False) + return self.not_match_return(self.recv_buf) self.recv_buf = self.recv_buf[length:] self.has_recv_header = True @@ -418,6 +419,7 @@ def __init__(self, method): self.client_id = 0 self.connection_id = 0 self.salt = b"auth_sha1_v2" + self.no_compatible_method = 'auth_sha1_v2' def init_data(self): return obfs_auth_v2_data() @@ -534,29 +536,24 @@ def server_post_decrypt(self, buf): return (b'', False) crc = struct.pack('H', self.recv_buf[4:6])[0] if length > len(self.recv_buf): return (b'', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] if sha1data != self.recv_buf[length - 10:length]: logging.error('auth_sha1_v2 data uncorrect auth HMAC-SHA1') - return (b'E', False) + return self.not_match_return(self.recv_buf) pos = common.ord(self.recv_buf[6]) if pos < 255: pos += 6 else: pos = struct.unpack('>H', self.recv_buf[7:9])[0] + 6 out_buf = self.recv_buf[pos:length - 10] - if len(out_buf) < 8: - self.raw_trans = True + if len(out_buf) < 12: self.recv_buf = b'' - logging.info('auth_sha1_v2: too short') - return (b'E', False) + logging.info('auth_sha1_v2: too short, data %s' % (binascii.hexlify(out_buf),)) + return self.not_match_return(self.recv_buf) client_id = struct.unpack(' Date: Tue, 16 Aug 2016 19:11:50 +0800 Subject: [PATCH 270/487] fix error return --- shadowsocks/obfsplugin/auth.py | 9 ++------- shadowsocks/obfsplugin/verify.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 8b673f37..c1321148 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -313,8 +313,7 @@ def server_post_decrypt(self, buf): pos = common.ord(self.recv_buf[6]) + 6 out_buf = self.recv_buf[pos:length - 10] if len(out_buf) < 12: - self.recv_buf = b'' - logging.info('auth_sha1: too short') + logging.info('auth_sha1: too short, data %s' % (binascii.hexlify(self.recv_buf),)) return self.not_match_return(self.recv_buf) utc_time = struct.unpack(' self.max_time_dif \ or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: - self.recv_buf = b'' logging.info('auth_sha1: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) return self.not_match_return(self.recv_buf) elif self.server_info.data.insert(client_id, connection_id): @@ -331,7 +329,6 @@ def server_post_decrypt(self, buf): self.client_id = client_id self.connection_id = connection_id else: - self.recv_buf = b'' logging.info('auth_sha1: auth fail, data %s' % (binascii.hexlify(out_buf),)) return self.not_match_return(self.recv_buf) self.recv_buf = self.recv_buf[length:] @@ -551,8 +548,7 @@ def server_post_decrypt(self, buf): pos = struct.unpack('>H', self.recv_buf[7:9])[0] + 6 out_buf = self.recv_buf[pos:length - 10] if len(out_buf) < 12: - self.recv_buf = b'' - logging.info('auth_sha1_v2: too short, data %s' % (binascii.hexlify(out_buf),)) + logging.info('auth_sha1_v2: too short, data %s' % (binascii.hexlify(self.recv_buf),)) return self.not_match_return(self.recv_buf) client_id = struct.unpack(' len(self.recv_buf): From 1e2445061ff085ac1a9c0cfbcd24688b96929361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 18 Aug 2016 09:31:12 +0800 Subject: [PATCH 271/487] remove keep alive --- shadowsocks/tcprelay.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a4e5c7b9..a6c981a9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -564,22 +564,6 @@ def _create_remote_socket(self, ip, port): remote_sock_v6.setblocking(False) else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - remote_sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - if self._is_local: - val_idle = 60 - val_intvl = 5 - else: - val_idle = 120 - val_intvl = 20 - if platform.system() in ['Linux']: - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, val_idle) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, val_intvl) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5) - elif platform.system() in ['Windows']: - remote_sock.ioctl(socket.SIO_KEEPALIVE_VALS, (1, 1000 * val_idle, 1000 * val_intvl)) - elif platform.system() in ["Darwin"]: #OSX - TCP_KEEPALIVE = 0x10 - remote_sock.setsockopt(socket.SOL_TCP, TCP_KEEPALIVE, val_intvl * 2) if not self._is_local: bind_addr = '' From e1aa68029dd2e3eace5ac40d3b2853eafe22d4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 20 Aug 2016 23:12:34 +0800 Subject: [PATCH 272/487] log if no user --- db_transfer.py | 4 ++++ shadowsocks/tcprelay.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index a9657b89..b85c66e4 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -311,6 +311,8 @@ def pull_db_all_user(self): rows = self.pull_db_users(conn) conn.close() + if len(rows) == 0: + logging.warn('no user in db') return rows def pull_db_users(self, conn): @@ -517,5 +519,7 @@ def pull_db_all_user(self): except Exception as e: logging.error(e) + if len(rows) == 0: + logging.warn('no user in json file') return rows diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a6c981a9..13e010aa 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -114,7 +114,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, config['method']) except Exception: self._stage = STAGE_DESTROYED - logging.error('creater encryptor fail at port %d', server._listen_port) + logging.error('create encryptor fail at port %d', server._listen_port) return self._encrypt_correct = True self._obfs = obfs.obfs(config['obfs']) From b3fcee662a2318e0868839b4eaa7e7ebbe004c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 20 Aug 2016 23:23:09 +0800 Subject: [PATCH 273/487] fix unit_len in auth_sha1 --- shadowsocks/obfsplugin/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index c1321148..ff87cb0a 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -188,7 +188,7 @@ class auth_sha1(verify_base): def __init__(self, method): super(auth_sha1, self).__init__(method) self.recv_buf = b'' - self.unit_len = 8100 + self.unit_len = 8000 self.decrypt_packet_num = 0 self.raw_trans = False self.has_sent_header = False From 5e485eb2bec7c9ace311d1292011aae1b5478218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 21 Aug 2016 23:07:33 +0800 Subject: [PATCH 274/487] warn if incorrect node id --- db_transfer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index b85c66e4..1e726ce2 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -311,7 +311,7 @@ def pull_db_all_user(self): rows = self.pull_db_users(conn) conn.close() - if len(rows) == 0: + if not rows: logging.warn('no user in db') return rows @@ -439,6 +439,7 @@ def pull_db_users(self, conn): rows = [] cur.close() conn.commit() + logging.warn('None result when select node info from ss_node in db, maybe you set the incorrect node id') return rows cur.close() @@ -519,7 +520,7 @@ def pull_db_all_user(self): except Exception as e: logging.error(e) - if len(rows) == 0: + if not rows: logging.warn('no user in json file') return rows From 6ee4eb3184861891969a254e0494566687507055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 21 Aug 2016 23:13:00 +0800 Subject: [PATCH 275/487] set default node id to 0 and default db to "sspanel" --- mysql.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql.json b/mysql.json index 3eb3bdad..1849e9e8 100644 --- a/mysql.json +++ b/mysql.json @@ -3,8 +3,8 @@ "port": 3306, "user": "ss", "password": "pass", - "db": "shadowsocks", - "node_id": 1, + "db": "sspanel", + "node_id": 0, "transfer_mul": 1.0, "ssl_enable": 0, "ssl_ca": "", From 6c34c4116fe694b597e6fdee78f6a1546d55473c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 22 Aug 2016 19:10:50 +0800 Subject: [PATCH 276/487] fix verify_sha1 response --- shadowsocks/obfsplugin/verify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index e4d24efd..71e01cc5 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -316,9 +316,9 @@ def server_post_decrypt(self, buf): else: self.raw_trans = True return (self.recv_buf, False) - head_size = self.get_head_size(self.recv_buf, 30) + head_size = self.get_head_size(self.recv_buf, 65536) if len(self.recv_buf) < head_size + 10: - return (b'', False) + return (b'E', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:head_size], hashlib.sha1).digest()[:10] if sha1data != self.recv_buf[head_size:head_size + 10]: logging.error('server_post_decrype data uncorrect auth HMAC-SHA1') From 3d4356e8b4012ef13c34ace68f8cd11bccfb7612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 22 Aug 2016 19:20:38 +0800 Subject: [PATCH 277/487] fix verify_sha1 redirect --- shadowsocks/obfsplugin/verify.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 71e01cc5..c37892a8 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -301,6 +301,12 @@ def client_post_decrypt(self, buf): def server_pre_encrypt(self, buf): return buf + def not_match_return(self, buf): + self.raw_trans = True + if self.method == 'verify_sha1': + return (b'E'*64, False) + return (buf, False) + def server_post_decrypt(self, buf): if self.raw_trans: return (buf, False) @@ -310,19 +316,14 @@ def server_post_decrypt(self, buf): if len(self.recv_buf) < 2: return (b'', False) if (ord(self.recv_buf[0]) & 0x10) != 0x10: - if self.method == 'verify_sha1': - logging.error('Not One-time authentication header') - return (b'E', False) - else: - self.raw_trans = True - return (self.recv_buf, False) + return self.not_match_return(self.recv_buf) head_size = self.get_head_size(self.recv_buf, 65536) if len(self.recv_buf) < head_size + 10: - return (b'E', False) + return self.not_match_return(self.recv_buf) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:head_size], hashlib.sha1).digest()[:10] if sha1data != self.recv_buf[head_size:head_size + 10]: logging.error('server_post_decrype data uncorrect auth HMAC-SHA1') - return (b'E', False) + return self.not_match_return(self.recv_buf) out_buf = to_bytes(chr(ord(self.recv_buf[0]) & 0xEF)) + self.recv_buf[1:head_size] self.recv_buf = self.recv_buf[head_size + 10:] self.has_recv_header = True From 1221a1f3f6c3d7206b4532eca62a99d8ce185e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 24 Aug 2016 17:11:01 +0800 Subject: [PATCH 278/487] log RLIMIT_NOFILE let http_simple the same as http_post when server_decode --- db_transfer.py | 2 ++ shadowsocks/obfsplugin/auth.py | 8 +++++-- shadowsocks/obfsplugin/http_simple.py | 33 +-------------------------- 3 files changed, 9 insertions(+), 34 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 1e726ce2..3c567dbe 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -189,6 +189,8 @@ def thread_db(obj): socket.setdefaulttimeout(timeout) last_rows = [] db_instance = obj() + import resource + logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) try: while True: load_config() diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index ff87cb0a..012f658a 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -298,12 +298,14 @@ def server_post_decrypt(self, buf): self.recv_buf += buf out_buf = b'' if not self.has_recv_header: - if len(self.recv_buf) < 4: + if len(self.recv_buf) < 6: return (b'', False) crc = struct.pack('H', self.recv_buf[4:6])[0] + if length > 2048: + return self.not_match_return(self.recv_buf) if length > len(self.recv_buf): return (b'', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] @@ -529,12 +531,14 @@ def server_post_decrypt(self, buf): self.recv_buf += buf out_buf = b'' if not self.has_recv_header: - if len(self.recv_buf) < 4: + if len(self.recv_buf) < 6: return (b'', False) crc = struct.pack('H', self.recv_buf[4:6])[0] + if length > 2048: + return self.not_match_return(self.recv_buf) if length > len(self.recv_buf): return (b'', False) sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 42715fcf..8701bb45 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -167,7 +167,7 @@ def server_decode(self, buf): self.recv_buffer += buf buf = self.recv_buffer if len(buf) > 10: - if match_begin(buf, b'GET /') or match_begin(buf, b'POST /'): + if match_begin(buf, b'GET ') or match_begin(buf, b'POST '): if len(buf) > 65536: self.recv_buffer = None logging.warn('http_simple: over size') @@ -238,37 +238,6 @@ def not_match_return(self, buf): return (b'E'*64, False, False) return (buf, True, False) - def server_decode(self, buf): - if self.has_recv_header: - return (buf, True, False) - - self.recv_buffer += buf - buf = self.recv_buffer - if len(buf) > 10: - if match_begin(buf, b'GET ') or match_begin(buf, b'POST '): - if len(buf) > 65536: - self.recv_buffer = None - logging.warn('http_post: over size') - return self.not_match_return(buf) - else: #not http header, run on original protocol - self.recv_buffer = None - logging.debug('http_post: not match begin') - return self.not_match_return(buf) - else: - return (b'', True, False) - - if b'\r\n\r\n' in buf: - datas = buf.split(b'\r\n\r\n', 1) - ret_buf = self.get_data_from_http_header(buf) - if len(datas) > 1: - ret_buf += datas[1] - if len(ret_buf) >= 7: - self.has_recv_header = True - return (ret_buf, True, False) - return self.not_match_return(buf) - else: - return (b'', True, False) - class random_head(plain.plain): def __init__(self, method): self.method = method From 1fe6e89c0752458b18acc343899323a88c44015d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 24 Aug 2016 17:18:02 +0800 Subject: [PATCH 279/487] fix log RLIMIT_NOFILE --- db_transfer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/db_transfer.py b/db_transfer.py index 3c567dbe..f7254352 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -189,6 +189,7 @@ def thread_db(obj): socket.setdefaulttimeout(timeout) last_rows = [] db_instance = obj() + ServerPool.get_instance() import resource logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) try: From e81f8db0b8e2baec15b8db53d9b13ddd5647f681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 31 Aug 2016 18:04:52 +0800 Subject: [PATCH 280/487] add auth_sha1_v3, remove tls1.0_session_auth --- mujson_mgr.py | 32 ++++- shadowsocks/obfsplugin/auth.py | 216 ++++++++++++++++++++++++++++- shadowsocks/obfsplugin/obfs_tls.py | 147 -------------------- 3 files changed, 240 insertions(+), 155 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index af7befc4..e106a9a9 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -92,7 +92,7 @@ def rand_pass(self): def add(self, user): up = {'enable': True, 'u': 0, 'd': 0, 'method': "aes-128-cfb", - 'protocol': "auth_sha1_v2_compatible", + 'protocol': "auth_sha1_v3_compatible", 'obfs': "tls1.2_ticket_auth_compatible", 'transfer_enable': 1125899906842624} up['passwd'] = self.rand_pass() @@ -206,17 +206,32 @@ def main(): fast_set_obfs = {'0': 'plain', '1': 'http_simple_compatible', '-1': 'http_simple', - '2': 'http_post_compatible', - '-2': 'http_post', - '3': 'tls1.2_ticket_auth_compatible', - '-3': 'tls1.2_ticket_auth'} + '2': 'tls1.2_ticket_auth_compatible', + '-2': 'tls1.2_ticket_auth'} fast_set_protocol = {'0': 'origin', '1': 'verify_sha1_compatible', '-1': 'verify_sha1', '2': 'auth_sha1_compatible', '-2': 'auth_sha1', '3': 'auth_sha1_v2_compatible', - '-3': 'auth_sha1_v2'} + '-3': 'auth_sha1_v2', + '4': 'auth_sha1_v3_compatible', + '-4': 'auth_sha1_v3'} + fast_set_method = {'a0': 'aes-128-cfb', + 'a1': 'aes-192-cfb', + 'a2': 'aes-256-cfb', + 'r': 'rc4-md5', + 'r6': 'rc4-md5-6', + 'c': 'chacha20', + 'ci': 'chacha20-ietf', + 's': 'salsa20', + 'b': 'bf-cfb', + 'm0': 'camellia-128-cfb', + 'm1': 'camellia-192-cfb', + 'm2': 'camellia-256-cfb', + 'a0t': 'aes-128-ctr', + 'a1t': 'aes-192-ctr', + 'a2t': 'aes-256-ctr'} try: optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) for key, value in optlist: @@ -251,7 +266,10 @@ def main(): elif key == '-G': user['protocol_param'] = value elif key == '-m': - user['method'] = value + if value in fast_set_method: + user['method'] = fast_set_method[value] + else: + user['method'] = value elif key == '-f': user['forbidden_port'] = value elif key == '-t': diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 012f658a..23b7cca6 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -42,11 +42,16 @@ def create_auth_sha1(method): def create_auth_sha1_v2(method): return auth_sha1_v2(method) +def create_auth_sha1_v3(method): + return auth_sha1_v3(method) + obfs_map = { 'auth_sha1': (create_auth_sha1,), 'auth_sha1_compatible': (create_auth_sha1,), 'auth_sha1_v2': (create_auth_sha1_v2,), 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), + 'auth_sha1_v3': (create_auth_sha1_v3,), + 'auth_sha1_v3_compatible': (create_auth_sha1_v3,), } def match_begin(str1, str2): @@ -582,7 +587,216 @@ def server_post_decrypt(self, buf): break if struct.pack('H', self.recv_buf[3:5])[0] + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + if pos == length - 4: + sendback = True + + if out_buf: + self.server_info.data.update(self.client_id, self.connection_id) + self.decrypt_packet_num += 1 + return (out_buf, sendback) + +class auth_sha1_v3(verify_base): + def __init__(self, method): + super(auth_sha1_v3, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + self.max_time_dif = 60 * 60 * 24 # time dif (second) setting + self.salt = b"auth_sha1_v3" + self.no_compatible_method = 'auth_sha1_v3' + + def init_data(self): + return obfs_auth_v2_data() + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + + def rnd_data(self, buf_size): + if buf_size > 1200: + return b'\x01' + + if buf_size > 400: + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 256) + else: + rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) + + if len(rnd_data) < 128: + return common.chr(len(rnd_data) + 1) + rnd_data + else: + return common.chr(255) + struct.pack('>H', len(rnd_data) + 3) + rnd_data + + def pack_data(self, buf): + data = self.rnd_data(len(buf)) + buf + data = struct.pack('>H', len(data) + 6) + data + adler32 = zlib.adler32(data) & 0xFFFFFFFF + data += struct.pack('H', data_len)) & 0xFFFFFFFF + data = struct.pack('H', data_len) + data + data += hmac.new(self.server_info.iv + self.server_info.key, data, hashlib.sha1).digest()[:10] + return data + + def auth_data(self): + utc_time = int(time.time()) & 0xFFFFFFFF + if self.server_info.data.connection_id > 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data error') + if length > len(self.recv_buf): + break + + if struct.pack('H', self.recv_buf[3:5])[0] + 2 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf + + def server_pre_encrypt(self, buf): + if self.raw_trans: + return buf + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return (buf, False) + self.recv_buf += buf + out_buf = b'' + if not self.has_recv_header: + if len(self.recv_buf) < 6: + return (b'', False) + crc = struct.pack('H', self.recv_buf[:2])[0] + if length > len(self.recv_buf): + return (b'', False) + sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] + if sha1data != self.recv_buf[length - 10:length]: + logging.error('auth_sha1_v3 data uncorrect auth HMAC-SHA1') + return self.not_match_return(self.recv_buf) + pos = common.ord(self.recv_buf[6]) + if pos < 255: + pos += 6 + else: + pos = struct.unpack('>H', self.recv_buf[7:9])[0] + 6 + out_buf = self.recv_buf[pos:length - 10] + if len(out_buf) < 12: + logging.info('auth_sha1_v3: too short, data %s' % (binascii.hexlify(self.recv_buf),)) + return self.not_match_return(self.recv_buf) + utc_time = struct.unpack(' self.max_time_dif: + logging.info('auth_sha1_v3: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) + return self.not_match_return(self.recv_buf) + elif self.server_info.data.insert(client_id, connection_id): + self.has_recv_header = True + out_buf = out_buf[12:] + self.client_id = client_id + self.connection_id = connection_id + else: + logging.info('auth_sha1_v3: auth fail, data %s' % (binascii.hexlify(out_buf),)) + return self.not_match_return(self.recv_buf) + self.recv_buf = self.recv_buf[length:] + self.has_recv_header = True + + sendback = False + while len(self.recv_buf) > 2: + length = struct.unpack('>H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + logging.info('auth_sha1_v3: over size') + return (b'E', False) + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if struct.pack('I', utc_time) + os.urandom(18) - data += hmac.new(self.server_info.key + client_id, data, hashlib.sha1).digest()[:10] - return data - - def client_encode(self, buf): - if self.raw_trans_sent: - return buf - self.send_buffer += buf - if not self.has_sent_header: - self.has_sent_header = True - data = self.tls_version + self.pack_auth_data(self.server_info.data.client_id) + b"\x20" + self.server_info.data.client_id + binascii.unhexlify(b"0016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") - data = b"\x01\x00" + struct.pack('>H', len(data)) + data - data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data - return data - if self.has_recv_header: - data = b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished - data += hmac.new(self.server_info.key + self.server_info.data.client_id, data, hashlib.sha1).digest()[:10] - ret = data + self.send_buffer - self.send_buffer = b'' - self.raw_trans_sent = True - return ret - return b'' - - def client_decode(self, buf): - if self.has_recv_header: - return (buf, False) - if len(buf) < 11 + 32 + 1 + 32: - raise Exception('client_decode data error') - verify = buf[11:33] - if hmac.new(self.server_info.key + self.server_info.data.client_id, verify, hashlib.sha1).digest()[:10] != buf[33:43]: - raise Exception('client_decode data error') - self.has_recv_header = True - return (b'', True) - - def server_encode(self, buf): - if self.raw_trans_sent or self.has_sent_header: - return buf - self.has_sent_header = True - data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"0016c02bc02fc00ac009c013c01400330039002f0035000a0100006fff01000100000a00080006001700180019000b0002010000230000337400000010002900270568322d31360568322d31350568322d313402683208737064792f332e3108687474702f312e31000500050100000000000d001600140401050106010201040305030603020304020202") - data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello - data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data - data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished - data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] - return data - - def decode_error_return(self, buf): - self.raw_trans_sent = True - self.raw_trans_recv = True - if self.method == 'tls1.0_session_auth': - return (b'E'*64, False, False) - return (buf, True, False) - - def server_decode(self, buf): - if self.raw_trans_recv: - return (buf, True, False) - - if self.has_recv_header: - verify = buf - verify_len = 43 - 10 - if len(buf) < 43: - raise Exception('server_decode data error') - if not match_begin(buf, b"\x14" + self.tls_version + "\x00\x01\x01"): #ChangeCipherSpec - raise Exception('server_decode data error') - buf = buf[6:] - if not match_begin(buf, b"\x16" + self.tls_version + "\x00\x20"): #Finished - raise Exception('server_decode data error') - if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: - raise Exception('server_decode data error') - if len(buf) < 37: - raise Exception('server_decode data error') - buf = buf[37:] - self.raw_trans_recv = True - return (buf, True, False) - - self.has_recv_header = True - ogn_buf = buf - if not match_begin(buf, b'\x16' + self.tls_version): - return self.decode_error_return(ogn_buf) - buf = buf[3:] - if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: - return self.decode_error_return(ogn_buf) - buf = buf[2:] - if not match_begin(buf, b'\x01\x00'): #client hello - return self.decode_error_return(ogn_buf) - buf = buf[2:] - if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: - return self.decode_error_return(ogn_buf) - buf = buf[2:] - if not match_begin(buf, self.tls_version): - return self.decode_error_return(ogn_buf) - buf = buf[2:] - verifyid = buf[:32] - buf = buf[32:] - sessionid_len = ord(buf[0]) - if sessionid_len < 32: - logging.info("tls_auth wrong sessionid_len") - return self.decode_error_return(ogn_buf) - sessionid = buf[1:sessionid_len + 1] - buf = buf[sessionid_len+1:] - self.client_id = sessionid - sha1 = hmac.new(self.server_info.key + sessionid, verifyid[:22], hashlib.sha1).digest()[:10] - utc_time = struct.unpack('>I', verifyid[:4])[0] - time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time) - if time_dif < -self.max_time_dif or time_dif > self.max_time_dif \ - or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: - logging.info("tls_auth wrong time") - return self.decode_error_return(ogn_buf) - if sha1 != verifyid[22:]: - logging.info("tls_auth wrong sha1") - return self.decode_error_return(ogn_buf) - if self.server_info.data.client_data.get(verifyid[:22]): - logging.info("replay attack detect, id = %s" % (binascii.hexlify(verifyid))) - return self.decode_error_return(ogn_buf) - self.server_info.data.client_data.sweep() - self.server_info.data.client_data[verifyid[:22]] = sessionid - # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) - return (b'', False, True) - class tls_ticket_auth(plain.plain): def __init__(self, method): self.method = method From 88017d964373bd045231049ff2eb5aa66543827f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 4 Sep 2016 11:23:46 +0800 Subject: [PATCH 281/487] fix ssl connection --- db_transfer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index f7254352..f31a5d79 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -286,7 +286,7 @@ def update_all_user(self, dt_transfer): conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], user=self.cfg["user"], passwd=self.cfg["password"], db=self.cfg["db"], charset='utf8', - ssl={'ca':self.cfg["ssl_enable"],'cert':self.cfg["ssl_enable"],'key':self.cfg["ssl_enable"]}) + ssl={'ca':self.cfg["ssl_ca"],'cert':self.cfg["ssl_cert"],'key':self.cfg["ssl_key"]}) else: conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], user=self.cfg["user"], passwd=self.cfg["password"], @@ -306,7 +306,7 @@ def pull_db_all_user(self): conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], user=self.cfg["user"], passwd=self.cfg["password"], db=self.cfg["db"], charset='utf8', - ssl={'ca':self.cfg["ssl_enable"],'cert':self.cfg["ssl_enable"],'key':self.cfg["ssl_enable"]}) + ssl={'ca':self.cfg["ssl_ca"],'cert':self.cfg["ssl_cert"],'key':self.cfg["ssl_key"]}) else: conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], user=self.cfg["user"], passwd=self.cfg["password"], @@ -361,7 +361,7 @@ def update_all_user(self, dt_transfer): conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], user=self.cfg["user"], passwd=self.cfg["password"], db=self.cfg["db"], charset='utf8', - ssl={'ca':self.cfg["ssl_enable"],'cert':self.cfg["ssl_enable"],'key':self.cfg["ssl_enable"]}) + ssl={'ca':self.cfg["ssl_ca"],'cert':self.cfg["ssl_cert"],'key':self.cfg["ssl_key"]}) else: conn = cymysql.connect(host=self.cfg["host"], port=self.cfg["port"], user=self.cfg["user"], passwd=self.cfg["password"], From 2637c8d1c72425508cec6f6998dd851027f77785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 4 Sep 2016 20:06:56 +0800 Subject: [PATCH 282/487] fix chacha20-ietf --- shadowsocks/crypto/sodium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/sodium.py b/shadowsocks/crypto/sodium.py index c4b5c1c9..51d476be 100644 --- a/shadowsocks/crypto/sodium.py +++ b/shadowsocks/crypto/sodium.py @@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function, \ with_statement -from ctypes import c_char_p, c_int, c_ulonglong, byref, \ +from ctypes import c_char_p, c_int, c_ulong, c_ulonglong, byref, \ create_string_buffer, c_void_p from shadowsocks.crypto import util From a02e995a8a36a432b9ce2720672ae28d9047f3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=8F=E4=BE=AF?= Date: Fri, 16 Sep 2016 19:26:54 +0800 Subject: [PATCH 283/487] fix bug at KqueueLoop close (#117) --- shadowsocks/eventloop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 2d7e696e..2ccd7e6c 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -99,7 +99,7 @@ def modify(self, fd, mode): self.register(fd, mode) def close(self): - self.kqueue.close() + self._kqueue.close() class SelectLoop(object): From 933e75b9a7b1d36c0bdca7f6f822cc944685c3c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 20 Sep 2016 17:21:32 +0800 Subject: [PATCH 284/487] add auth_sha1_v4 --- shadowsocks/obfsplugin/auth.py | 229 +++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 23b7cca6..10aff0ba 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -45,6 +45,9 @@ def create_auth_sha1_v2(method): def create_auth_sha1_v3(method): return auth_sha1_v3(method) +def create_auth_sha1_v4(method): + return auth_sha1_v4(method) + obfs_map = { 'auth_sha1': (create_auth_sha1,), 'auth_sha1_compatible': (create_auth_sha1,), @@ -52,6 +55,8 @@ def create_auth_sha1_v3(method): 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), 'auth_sha1_v3': (create_auth_sha1_v3,), 'auth_sha1_v3_compatible': (create_auth_sha1_v3,), + 'auth_sha1_v4': (create_auth_sha1_v4,), + 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), } def match_begin(str1, str2): @@ -819,3 +824,227 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, sendback) +class auth_sha1_v4(verify_base): + def __init__(self, method): + super(auth_sha1_v4, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.decrypt_packet_num = 0 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + self.max_time_dif = 60 * 60 * 24 # time dif (second) setting + self.salt = b"auth_sha1_v4" + self.no_compatible_method = 'auth_sha1_v4' + + def init_data(self): + return obfs_auth_v2_data() + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + + def rnd_data(self, buf_size): + if buf_size > 1200: + return b'\x01' + + if buf_size > 400: + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 256) + else: + rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) + + if len(rnd_data) < 128: + return common.chr(len(rnd_data) + 1) + rnd_data + else: + return common.chr(255) + struct.pack('>H', len(rnd_data) + 3) + rnd_data + + def pack_data(self, buf): + data = self.rnd_data(len(buf)) + buf + data_len = len(data) + 8 + crc = binascii.crc32(struct.pack('>H', data_len)) & 0xFFFF + data = struct.pack('H', data_len) + data + adler32 = zlib.adler32(data) & 0xFFFFFFFF + data += struct.pack('H', data_len) + self.salt + self.server_info.key) & 0xFFFFFFFF + data = struct.pack('H', data_len) + data + data += hmac.new(self.server_info.iv + self.server_info.key, data, hashlib.sha1).digest()[:10] + return data + + def auth_data(self): + utc_time = int(time.time()) & 0xFFFFFFFF + if self.server_info.data.connection_id > 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 4: + crc = struct.pack('H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data error') + if length > len(self.recv_buf): + break + + if struct.pack('H', self.recv_buf[5:7])[0] + 4 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + + if out_buf: + self.decrypt_packet_num += 1 + return out_buf + + def server_pre_encrypt(self, buf): + if self.raw_trans: + return buf + ret = b'' + while len(buf) > self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return (buf, False) + self.recv_buf += buf + out_buf = b'' + if not self.has_recv_header: + if len(self.recv_buf) <= 6: + return (b'', False) + crc = struct.pack('H', self.recv_buf[:2])[0] + if length > len(self.recv_buf): + return (b'', False) + sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] + if sha1data != self.recv_buf[length - 10:length]: + logging.error('auth_sha1_v4 data uncorrect auth HMAC-SHA1') + return self.not_match_return(self.recv_buf) + pos = common.ord(self.recv_buf[6]) + if pos < 255: + pos += 6 + else: + pos = struct.unpack('>H', self.recv_buf[7:9])[0] + 6 + out_buf = self.recv_buf[pos:length - 10] + if len(out_buf) < 12: + logging.info('auth_sha1_v4: too short, data %s' % (binascii.hexlify(self.recv_buf),)) + return self.not_match_return(self.recv_buf) + utc_time = struct.unpack(' self.max_time_dif: + logging.info('auth_sha1_v4: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) + return self.not_match_return(self.recv_buf) + elif self.server_info.data.insert(client_id, connection_id): + self.has_recv_header = True + out_buf = out_buf[12:] + self.client_id = client_id + self.connection_id = connection_id + else: + logging.info('auth_sha1_v4: auth fail, data %s' % (binascii.hexlify(out_buf),)) + return self.not_match_return(self.recv_buf) + self.recv_buf = self.recv_buf[length:] + self.has_recv_header = True + + sendback = False + while len(self.recv_buf) > 4: + crc = struct.pack('H', self.recv_buf[:2])[0] + if length >= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.decrypt_packet_num == 0: + logging.info('auth_sha1_v4: over size') + return (b'E', False) + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if struct.pack('H', self.recv_buf[5:7])[0] + 4 + out_buf += self.recv_buf[pos:length - 4] + self.recv_buf = self.recv_buf[length:] + if pos == length - 4: + sendback = True + + if out_buf: + self.server_info.data.update(self.client_id, self.connection_id) + self.decrypt_packet_num += 1 + return (out_buf, sendback) + From 52041387f9ee9961e6e5fdb0099c48c8d6a9aa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 21 Sep 2016 15:16:50 +0800 Subject: [PATCH 285/487] fix push_db_all_user --- db_transfer.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index f31a5d79..de0a481d 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -41,16 +41,14 @@ def push_db_all_user(self): dt_transfer[id] = [self.last_get_transfer[id][0] - last_transfer[id][0], self.last_get_transfer[id][1] - last_transfer[id][1]] for id in curr_transfer.keys(): + if id in self.force_update_transfer: + continue #算出与上次记录的流量差值，保存于dt_transfer表 if id in last_transfer: if curr_transfer[id][0] + curr_transfer[id][1] - last_transfer[id][0] - last_transfer[id][1] <= 0: continue - if last_transfer[id][0] <= curr_transfer[id][0] and \ - last_transfer[id][1] <= curr_transfer[id][1]: - dt_transfer[id] = [curr_transfer[id][0] - last_transfer[id][0], - curr_transfer[id][1] - last_transfer[id][1]] - else: - dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]] + dt_transfer[id] = [curr_transfer[id][0] - last_transfer[id][0], + curr_transfer[id][1] - last_transfer[id][1]] else: if curr_transfer[id][0] + curr_transfer[id][1] <= 0: continue @@ -67,14 +65,16 @@ def push_db_all_user(self): update_transfer = self.update_all_user(dt_transfer) #返回有更新的表 for id in update_transfer.keys(): #其增量加在此表 - if id in self.force_update_transfer: #但排除在force_update_transfer内的 - if id in self.last_update_transfer: - del self.last_update_transfer[id] - self.force_update_transfer.remove(id) - else: + if id not in self.force_update_transfer: #但排除在force_update_transfer内的 last = self.last_update_transfer.get(id, [0,0]) self.last_update_transfer[id] = [last[0] + update_transfer[id][0], last[1] + update_transfer[id][1]] self.last_get_transfer = curr_transfer + for id in self.force_update_transfer: + if id in self.last_update_transfer: + del self.last_update_transfer[id] + if id in self.last_get_transfer: + del self.last_get_transfer[id] + self.force_update_transfer = set() def del_server_out_of_bound_safe(self, last_rows, rows): #停止超流量的服务 @@ -117,6 +117,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): if not allow: logging.info('db stop server at port [%s]' % (port,)) ServerPool.get_instance().cb_del_server(port) + self.force_update_transfer.add(port) else: cfgchange = False if port in ServerPool.get_instance().tcp_servers_pool: @@ -135,6 +136,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): if cfgchange: logging.info('db stop server at port [%s] reason: config changed: %s' % (port, cfg)) ServerPool.get_instance().cb_del_server(port) + self.force_update_transfer.add(port) new_servers[port] = (passwd, cfg) elif allow and ServerPool.get_instance().server_run_status(port) is False: @@ -146,6 +148,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): else: logging.info('db stop server at port [%s] reason: port not exist' % (row['port'])) ServerPool.get_instance().cb_del_server(row['port']) + self.clear_cache(row['port']) if row['port'] in self.port_uid_table: del self.port_uid_table[row['port']] @@ -156,13 +159,17 @@ def del_server_out_of_bound_safe(self, last_rows, rows): passwd, cfg = new_servers[port] self.new_server(port, passwd, cfg) + def clear_cache(self, port): + if port in self.force_update_transfer: del self.force_update_transfer[port] + if port in self.last_get_transfer: del self.last_get_transfer[port] + if port in self.last_update_transfer: del self.last_update_transfer[port] + def new_server(self, port, passwd, cfg): protocol = cfg.get('protocol', ServerPool.get_instance().config.get('protocol', 'origin')) method = cfg.get('method', ServerPool.get_instance().config.get('method', 'None')) obfs = cfg.get('obfs', ServerPool.get_instance().config.get('obfs', 'plain')) logging.info('db start server at port [%s] pass [%s] protocol [%s] method [%s] obfs [%s]' % (port, passwd, protocol, method, obfs)) ServerPool.get_instance().new_server(port, cfg) - self.force_update_transfer.add(port) def cmp(self, val1, val2): if type(val1) is bytes: @@ -230,7 +237,7 @@ def __init__(self): "user": "ss", "password": "pass", "db": "shadowsocks", - "node_id": 1, + "node_id": 0, "transfer_mul": 1.0, "ssl_enable": 0, "ssl_ca": "", @@ -261,8 +268,9 @@ def update_all_user(self, dt_transfer): for id in dt_transfer.keys(): transfer = dt_transfer[id] #小于最低更新流量的先不更新 - update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) - if transfer[0] + transfer[1] < update_trs: + update_trs = 1024 * (2048 - self.user_pass.get(id, 0) * 64) + if transfer[0] + transfer[1] < update_trs and id not in self.force_update_transfer: + self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue if id in self.user_pass: del self.user_pass[id] @@ -372,7 +380,7 @@ def update_all_user(self, dt_transfer): transfer = dt_transfer[id] bandwidth_thistime = bandwidth_thistime + transfer[0] + transfer[1] - update_trs = 1024 * max(2048 - self.user_pass.get(id, 0) * 64, 16) + update_trs = 1024 * (2048 - self.user_pass.get(id, 0) * 64) if transfer[0] + transfer[1] < update_trs: self.user_pass[id] = self.user_pass.get(id, 0) + 1 continue From a407bcc5684d3c5475d1b39c226bed8c326eb3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 22 Sep 2016 15:07:01 +0800 Subject: [PATCH 286/487] add auth_aes128 --- config.json | 2 +- mujson_mgr.py | 25 +-- shadowsocks/crypto/openssl.py | 3 + shadowsocks/encrypt.py | 7 +- shadowsocks/obfsplugin/auth.py | 260 ++++++++++++++++++++++++++++++- shadowsocks/obfsplugin/verify.py | 5 +- shadowsocks/tcprelay.py | 12 +- 7 files changed, 291 insertions(+), 23 deletions(-) diff --git a/config.json b/config.json index 0a72c704..f30ce55c 100644 --- a/config.json +++ b/config.json @@ -8,7 +8,7 @@ "timeout": 120, "udp_timeout": 60, "method": "aes-256-cfb", - "protocol": "auth_sha1_compatible", + "protocol": "auth_sha1_v2_compatible", "protocol_param": "", "obfs": "http_simple_compatible", "obfs_param": "", diff --git a/mujson_mgr.py b/mujson_mgr.py index e106a9a9..df990a98 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -204,19 +204,20 @@ def main(): action = None user = {} fast_set_obfs = {'0': 'plain', - '1': 'http_simple_compatible', - '-1': 'http_simple', - '2': 'tls1.2_ticket_auth_compatible', - '-2': 'tls1.2_ticket_auth'} + '+1': 'http_simple_compatible', + '1': 'http_simple', + '+2': 'tls1.2_ticket_auth_compatible', + '2': 'tls1.2_ticket_auth'} fast_set_protocol = {'0': 'origin', - '1': 'verify_sha1_compatible', - '-1': 'verify_sha1', - '2': 'auth_sha1_compatible', - '-2': 'auth_sha1', - '3': 'auth_sha1_v2_compatible', - '-3': 'auth_sha1_v2', - '4': 'auth_sha1_v3_compatible', - '-4': 'auth_sha1_v3'} + '+1': 'verify_sha1_compatible', + '1': 'verify_sha1', + '+2': 'auth_sha1_compatible', + '2': 'auth_sha1', + '+3': 'auth_sha1_v2_compatible', + '3': 'auth_sha1_v2', + '+4': 'auth_sha1_v4_compatible', + '4': 'auth_sha1_v4', + 'a1': 'auth_aes128'} fast_set_method = {'a0': 'aes-128-cfb', 'a1': 'aes-192-cfb', 'a2': 'aes-256-cfb', diff --git a/shadowsocks/crypto/openssl.py b/shadowsocks/crypto/openssl.py index 7a4df2b1..630baefe 100644 --- a/shadowsocks/crypto/openssl.py +++ b/shadowsocks/crypto/openssl.py @@ -125,6 +125,9 @@ def clean(self): ciphers = { + 'aes-128-cbc': (16, 16, OpenSSLCrypto), + 'aes-192-cbc': (24, 16, OpenSSLCrypto), + 'aes-256-cbc': (32, 16, OpenSSLCrypto), 'aes-128-cfb': (16, 16, OpenSSLCrypto), 'aes-192-cfb': (24, 16, OpenSSLCrypto), 'aes-256-cfb': (32, 16, OpenSSLCrypto), diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index e8f51b74..44f90525 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -73,7 +73,7 @@ def EVP_BytesToKey(password, key_len, iv_len): class Encryptor(object): - def __init__(self, key, method): + def __init__(self, key, method, iv = None): self.key = key self.method = method self.iv = None @@ -85,8 +85,11 @@ def __init__(self, key, method): method = method.lower() self._method_info = self.get_method_info(method) if self._method_info: - self.cipher = self.get_cipher(key, method, 1, + if iv is None or len(iv) != self._method_info[1]: + self.cipher = self.get_cipher(key, method, 1, random_string(self._method_info[1])) + else: + self.cipher = self.get_cipher(key, method, 1, iv) else: logging.error('method %s not supported' % method) sys.exit(1) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 10aff0ba..bc719597 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -32,7 +32,7 @@ import hashlib import shadowsocks -from shadowsocks import common, lru_cache +from shadowsocks import common, lru_cache, encrypt from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord, chr @@ -48,6 +48,9 @@ def create_auth_sha1_v3(method): def create_auth_sha1_v4(method): return auth_sha1_v4(method) +def create_auth_aes128(method): + return auth_aes128(method) + obfs_map = { 'auth_sha1': (create_auth_sha1,), 'auth_sha1_compatible': (create_auth_sha1,), @@ -57,6 +60,7 @@ def create_auth_sha1_v4(method): 'auth_sha1_v3_compatible': (create_auth_sha1_v3,), 'auth_sha1_v4': (create_auth_sha1_v4,), 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), + 'auth_aes128': (create_auth_aes128,), } def match_begin(str1, str2): @@ -1048,3 +1052,257 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, sendback) +class auth_aes128(verify_base): + def __init__(self, method): + super(auth_aes128, self).__init__(method) + self.recv_buf = b'' + self.unit_len = 8100 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + self.max_time_dif = 60 * 60 * 24 # time dif (second) setting + self.salt = b"auth_aes128" + self.no_compatible_method = 'auth_aes128' + self.extra_wait_size = struct.unpack('>H', os.urandom(2))[0] % 1024 + self.pack_id = 0 + self.recv_id = 0 + + def init_data(self): + return obfs_auth_v2_data() + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + + def rnd_data(self, buf_size): + if buf_size > 1200: + return b'\x01' + + if self.pack_id > 4: + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 32) + elif buf_size > 400: + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) + else: + rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) + + if len(rnd_data) < 128: + return common.chr(len(rnd_data) + 1) + rnd_data + else: + return common.chr(255) + struct.pack(' 400: + rnd_len = common.ord(os.urandom(1)[0]) % 512 + else: + rnd_len = struct.unpack(' 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 4: + crc = struct.pack('= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data error') + if length > len(self.recv_buf): + break + + if struct.pack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return (buf, False) + self.recv_buf += buf + out_buf = b'' + if not self.has_recv_header: + if len(self.recv_buf) < 30: + return (b'', False) + sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:20], hashlib.sha1).digest()[:10] + if sha1data != self.recv_buf[20:30]: + logging.error('auth_aes128 data uncorrect auth HMAC-SHA1 from %s:%d, data %s' % (self.server_info.client, self.server_info.client_port, binascii.hexlify(self.recv_buf))) + if len(self.recv_buf) < 30 + self.extra_wait_size: + return (b'', False) + return self.not_match_return(self.recv_buf) + + user_key = self.recv_buf[:4] + encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(user_key + self.server_info.key)) + self.salt, 'aes-128-cbc') + head = encryptor.decrypt(b'\x00' * 16 + self.recv_buf[4:20] + b'\x00') # need an extra byte or recv empty + length = struct.unpack(' self.max_time_dif: + logging.info('auth_aes128: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(head),)) + return self.not_match_return(self.recv_buf) + elif self.server_info.data.insert(client_id, connection_id): + self.has_recv_header = True + out_buf = self.recv_buf[30 + rnd_len:length - 4] + self.client_id = client_id + self.connection_id = connection_id + else: + logging.info('auth_aes128: auth fail, data %s' % (binascii.hexlify(out_buf),)) + return self.not_match_return(self.recv_buf) + self.recv_buf = self.recv_buf[length:] + self.has_recv_header = True + + sendback = False + while len(self.recv_buf) > 4: + crc = struct.pack('= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.recv_id == 0: + logging.info('auth_aes128: over size') + return (b'E', False) + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if struct.pack(' Date: Mon, 26 Sep 2016 21:19:33 +0800 Subject: [PATCH 287/487] fix typo --- mujson_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index df990a98..d7a810a4 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -92,7 +92,7 @@ def rand_pass(self): def add(self, user): up = {'enable': True, 'u': 0, 'd': 0, 'method': "aes-128-cfb", - 'protocol': "auth_sha1_v3_compatible", + 'protocol': "auth_sha1_v4_compatible", 'obfs': "tls1.2_ticket_auth_compatible", 'transfer_enable': 1125899906842624} up['passwd'] = self.rand_pass() From ada4eab628aef06f1505e1c3b304a739863c5a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 30 Sep 2016 10:12:24 +0800 Subject: [PATCH 288/487] sendback an empty packet if auth success --- shadowsocks/obfsplugin/auth.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index bc719597..bb7c238b 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -544,6 +544,8 @@ def server_post_decrypt(self, buf): return (buf, False) self.recv_buf += buf out_buf = b'' + sendback = False + if not self.has_recv_header: if len(self.recv_buf) < 6: return (b'', False) @@ -580,8 +582,8 @@ def server_post_decrypt(self, buf): return self.not_match_return(self.recv_buf) self.recv_buf = self.recv_buf[length:] self.has_recv_header = True + sendback = True - sendback = False while len(self.recv_buf) > 2: length = struct.unpack('>H', self.recv_buf[:2])[0] if length >= 8192 or length < 7: @@ -965,6 +967,8 @@ def server_post_decrypt(self, buf): return (buf, False) self.recv_buf += buf out_buf = b'' + sendback = False + if not self.has_recv_header: if len(self.recv_buf) <= 6: return (b'', False) @@ -1004,8 +1008,8 @@ def server_post_decrypt(self, buf): return self.not_match_return(self.recv_buf) self.recv_buf = self.recv_buf[length:] self.has_recv_header = True + sendback = True - sendback = False while len(self.recv_buf) > 4: crc = struct.pack(' 4: crc = struct.pack(' Date: Sat, 1 Oct 2016 21:10:27 +0800 Subject: [PATCH 289/487] fix bugs in auth_aes128 when run as a client add auth_aes128_sha1 --- shadowsocks/obfsplugin/auth.py | 293 +++++++++++++++++++++++++++++++-- 1 file changed, 282 insertions(+), 11 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index bb7c238b..bf5c2c72 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -51,6 +51,9 @@ def create_auth_sha1_v4(method): def create_auth_aes128(method): return auth_aes128(method) +def create_auth_aes128_sha1(method): + return auth_aes128_sha1(method) + obfs_map = { 'auth_sha1': (create_auth_sha1,), 'auth_sha1_compatible': (create_auth_sha1,), @@ -61,6 +64,8 @@ def create_auth_aes128(method): 'auth_sha1_v4': (create_auth_sha1_v4,), 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), 'auth_aes128': (create_auth_aes128,), + 'auth_aes128_sha1': (create_auth_aes128_sha1,), + 'auth_aes128_sha1_compatible': (create_auth_aes128_sha1,), } def match_begin(str1, str2): @@ -73,9 +78,9 @@ class obfs_verify_data(object): def __init__(self): pass -class verify_base(plain.plain): +class auth_base(plain.plain): def __init__(self, method): - super(verify_base, self).__init__(method) + super(auth_base, self).__init__(method) self.method = method self.no_compatible_method = '' @@ -198,7 +203,7 @@ def insert(self, client_id, connection_id): else: return self.client_id[client_id].insert(connection_id) -class auth_sha1(verify_base): +class auth_sha1(auth_base): def __init__(self, method): super(auth_sha1, self).__init__(method) self.recv_buf = b'' @@ -420,7 +425,7 @@ def insert(self, client_id, connection_id): else: return self.client_id[client_id].insert(connection_id) -class auth_sha1_v2(verify_base): +class auth_sha1_v2(auth_base): def __init__(self, method): super(auth_sha1_v2, self).__init__(method) self.recv_buf = b'' @@ -621,7 +626,7 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, sendback) -class auth_sha1_v3(verify_base): +class auth_sha1_v3(auth_base): def __init__(self, method): super(auth_sha1_v3, self).__init__(method) self.recv_buf = b'' @@ -830,7 +835,7 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, sendback) -class auth_sha1_v4(verify_base): +class auth_sha1_v4(auth_base): def __init__(self, method): super(auth_sha1_v4, self).__init__(method) self.recv_buf = b'' @@ -1056,7 +1061,7 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, sendback) -class auth_aes128(verify_base): +class auth_aes128(auth_base): def __init__(self, method): super(auth_aes128, self).__init__(method) self.recv_buf = b'' @@ -1119,15 +1124,14 @@ def pack_auth_data(self, auth_data, buf): else: rnd_len = struct.unpack('H', os.urandom(2))[0] % 1024 + self.pack_id = 0 + self.recv_id = 0 + + def init_data(self): + return obfs_auth_v2_data() + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + + def rnd_data(self, buf_size): + if buf_size > 1200: + return b'\x01' + + if self.pack_id > 4: + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 32) + elif buf_size > 400: + rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) + else: + rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) + + if len(rnd_data) < 128: + return common.chr(len(rnd_data) + 1) + rnd_data + else: + return common.chr(255) + struct.pack(' 400: + rnd_len = struct.unpack(' 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 4: + mac_key = self.server_info.key + struct.pack('= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data error') + if length > len(self.recv_buf): + break + + if hmac.new(mac_key, self.recv_buf[:length - 4], hashlib.sha1).digest()[:4] != self.recv_buf[length - 4:length]: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data uncorrect checksum') + + self.recv_id = (self.recv_id + 1) & 0xFFFFFFFF + pos = common.ord(self.recv_buf[4]) + if pos < 255: + pos += 4 + else: + pos = struct.unpack(' self.unit_len: + ret += self.pack_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return (buf, False) + self.recv_buf += buf + out_buf = b'' + sendback = False + + if not self.has_recv_header: + if len(self.recv_buf) < 7: + return (b'', False) + mac_key = self.server_info.recv_iv + self.server_info.key + sha1data = hmac.new(mac_key, self.recv_buf[:3], hashlib.sha1).digest()[:4] + if sha1data != self.recv_buf[3:7]: + return self.not_match_return(self.recv_buf) + + if len(self.recv_buf) < 31: + return (b'', False) + sha1data = hmac.new(mac_key, self.recv_buf[7:27], hashlib.sha1).digest()[:4] + if sha1data != self.recv_buf[27:31]: + logging.error('auth_aes128_sha1 data uncorrect auth HMAC-SHA1 from %s:%d, data %s' % (self.server_info.client, self.server_info.client_port, binascii.hexlify(self.recv_buf))) + if len(self.recv_buf) < 31 + self.extra_wait_size: + return (b'', False) + return self.not_match_return(self.recv_buf) + + user_key = self.recv_buf[7:11] + encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(user_key + self.server_info.key)) + self.salt, 'aes-128-cbc') + head = encryptor.decrypt(b'\x00' * 16 + self.recv_buf[11:27] + b'\x00') # need an extra byte or recv empty + length = struct.unpack(' self.max_time_dif: + logging.info('auth_aes128_sha1: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(head),)) + return self.not_match_return(self.recv_buf) + elif self.server_info.data.insert(client_id, connection_id): + self.has_recv_header = True + out_buf = self.recv_buf[31 + rnd_len:length - 4] + self.client_id = client_id + self.connection_id = connection_id + else: + logging.info('auth_aes128_sha1: auth fail, data %s' % (binascii.hexlify(out_buf),)) + return self.not_match_return(self.recv_buf) + self.recv_buf = self.recv_buf[length:] + self.has_recv_header = True + sendback = True + + while len(self.recv_buf) > 4: + mac_key = self.server_info.key + struct.pack('= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + if self.recv_id == 0: + logging.info('auth_aes128_sha1: over size') + return (b'E', False) + else: + raise Exception('server_post_decrype data error') + if length > len(self.recv_buf): + break + + if hmac.new(mac_key, self.recv_buf[:length - 4], hashlib.sha1).digest()[:4] != self.recv_buf[length - 4:length]: + logging.info('auth_aes128_sha1: checksum error, data %s' % (binascii.hexlify(self.recv_buf[:length]),)) + self.raw_trans = True + self.recv_buf = b'' + if self.recv_id == 0: + return (b'E', False) + else: + raise Exception('server_post_decrype data uncorrect checksum') + + self.recv_id = (self.recv_id + 1) & 0xFFFFFFFF + pos = common.ord(self.recv_buf[4]) + if pos < 255: + pos += 4 + else: + pos = struct.unpack(' Date: Mon, 3 Oct 2016 00:17:08 +0800 Subject: [PATCH 290/487] add user key, add HMAC-MD5 --- shadowsocks/obfsplugin/auth.py | 100 +++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index bf5c2c72..f96ab3ba 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -51,8 +51,11 @@ def create_auth_sha1_v4(method): def create_auth_aes128(method): return auth_aes128(method) +def create_auth_aes128_md5(method): + return auth_aes128_sha1(method, hashlib.md5) + def create_auth_aes128_sha1(method): - return auth_aes128_sha1(method) + return auth_aes128_sha1(method, hashlib.sha1) obfs_map = { 'auth_sha1': (create_auth_sha1,), @@ -64,6 +67,8 @@ def create_auth_aes128_sha1(method): 'auth_sha1_v4': (create_auth_sha1_v4,), 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), 'auth_aes128': (create_auth_aes128,), + 'auth_aes128_md5': (create_auth_aes128_md5,), + 'auth_aes128_md5_compatible': (create_auth_aes128_md5,), 'auth_aes128_sha1': (create_auth_aes128_sha1,), 'auth_aes128_sha1_compatible': (create_auth_aes128_sha1,), } @@ -1095,7 +1100,7 @@ def rnd_data(self, buf_size): if self.pack_id > 4: rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 32) - elif buf_size > 400: + elif buf_size > 900: rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) else: rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) @@ -1318,8 +1323,9 @@ def server_udp_post_decrypt(self, buf): return data class auth_aes128_sha1(auth_base): - def __init__(self, method): + def __init__(self, method, hashfunc): super(auth_aes128_sha1, self).__init__(method) + self.hashfunc = hashfunc self.recv_buf = b'' self.unit_len = 8100 self.raw_trans = False @@ -1328,11 +1334,12 @@ def __init__(self, method): self.client_id = 0 self.connection_id = 0 self.max_time_dif = 60 * 60 * 24 # time dif (second) setting - self.salt = b"auth_aes128_sha1" - self.no_compatible_method = 'auth_aes128_sha1' + self.salt = hashfunc == hashlib.md5 and b"auth_aes128_md5" or b"auth_aes128_sha1" + self.no_compatible_method = hashfunc == hashlib.md5 and "auth_aes128_md5" or 'auth_aes128_sha1' self.extra_wait_size = struct.unpack('>H', os.urandom(2))[0] % 1024 - self.pack_id = 0 - self.recv_id = 0 + self.pack_id = 1 + self.recv_id = 1 + self.user_key = None def init_data(self): return obfs_auth_v2_data() @@ -1351,7 +1358,7 @@ def rnd_data(self, buf_size): if self.pack_id > 4: rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 32) - elif buf_size > 400: + elif buf_size > 900: rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) else: rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) @@ -1364,10 +1371,10 @@ def rnd_data(self, buf_size): def pack_data(self, buf): data = self.rnd_data(len(buf)) + buf data_len = len(data) + 8 - mac_key = self.server_info.key + struct.pack(' 4: - mac_key = self.server_info.key + struct.pack(' len(self.recv_buf): break - if hmac.new(mac_key, self.recv_buf[:length - 4], hashlib.sha1).digest()[:4] != self.recv_buf[length - 4:length]: + if hmac.new(mac_key, self.recv_buf[:length - 4], self.hashfunc).digest()[:4] != self.recv_buf[length - 4:length]: self.raw_trans = True self.recv_buf = b'' raise Exception('client_post_decrypt data uncorrect checksum') @@ -1474,21 +1483,23 @@ def server_post_decrypt(self, buf): if len(self.recv_buf) < 7: return (b'', False) mac_key = self.server_info.recv_iv + self.server_info.key - sha1data = hmac.new(mac_key, self.recv_buf[:3], hashlib.sha1).digest()[:4] + sha1data = hmac.new(mac_key, self.recv_buf[:3], self.hashfunc).digest()[:4] if sha1data != self.recv_buf[3:7]: return self.not_match_return(self.recv_buf) if len(self.recv_buf) < 31: return (b'', False) - sha1data = hmac.new(mac_key, self.recv_buf[7:27], hashlib.sha1).digest()[:4] + sha1data = hmac.new(mac_key, self.recv_buf[7:27], self.hashfunc).digest()[:4] if sha1data != self.recv_buf[27:31]: - logging.error('auth_aes128_sha1 data uncorrect auth HMAC-SHA1 from %s:%d, data %s' % (self.server_info.client, self.server_info.client_port, binascii.hexlify(self.recv_buf))) + logging.error('%s data uncorrect auth HMAC-SHA1 from %s:%d, data %s' % (self.no_compatible_method, self.server_info.client, self.server_info.client_port, binascii.hexlify(self.recv_buf))) if len(self.recv_buf) < 31 + self.extra_wait_size: return (b'', False) return self.not_match_return(self.recv_buf) user_key = self.recv_buf[7:11] - encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(user_key + self.server_info.key)) + self.salt, 'aes-128-cbc') + #if user_key in user_map: self.user_key[user_key] else: # TODO + self.user_key = self.server_info.key + encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(self.user_key)) + self.salt, 'aes-128-cbc') head = encryptor.decrypt(b'\x00' * 16 + self.recv_buf[11:27] + b'\x00') # need an extra byte or recv empty length = struct.unpack(' self.max_time_dif: - logging.info('auth_aes128_sha1: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(head),)) + logging.info('%s: wrong timestamp, time_dif %d, data %s' % (self.no_compatible_method, time_dif, binascii.hexlify(head))) return self.not_match_return(self.recv_buf) elif self.server_info.data.insert(client_id, connection_id): self.has_recv_header = True @@ -1511,20 +1522,20 @@ def server_post_decrypt(self, buf): self.client_id = client_id self.connection_id = connection_id else: - logging.info('auth_aes128_sha1: auth fail, data %s' % (binascii.hexlify(out_buf),)) + logging.info('%s: auth fail, data %s' % (self.no_compatible_method, binascii.hexlify(out_buf))) return self.not_match_return(self.recv_buf) self.recv_buf = self.recv_buf[length:] self.has_recv_header = True sendback = True while len(self.recv_buf) > 4: - mac_key = self.server_info.key + struct.pack(' len(self.recv_buf): break - if hmac.new(mac_key, self.recv_buf[:length - 4], hashlib.sha1).digest()[:4] != self.recv_buf[length - 4:length]: - logging.info('auth_aes128_sha1: checksum error, data %s' % (binascii.hexlify(self.recv_buf[:length]),)) + if hmac.new(mac_key, self.recv_buf[:length - 4], self.hashfunc).digest()[:4] != self.recv_buf[length - 4:length]: + logging.info('%s: checksum error, data %s' % (self.no_compatible_method, binascii.hexlify(self.recv_buf[:length]))) self.raw_trans = True self.recv_buf = b'' if self.recv_id == 0: @@ -1565,21 +1576,24 @@ def server_post_decrypt(self, buf): return (out_buf, sendback) def client_udp_pre_encrypt(self, buf): - return buf + hmac.new(self.server_info.key, buf, hashlib.sha1).digest()[:4] + uid = os.urandom(4) + user_key = self.server_info.key + buf += uid + return buf + hmac.new(user_key, buf, self.hashfunc).digest()[:4] def client_udp_post_decrypt(self, buf): - length = len(buf) - data = buf[:-4] - if hmac.new(self.server_info.key, data, hashlib.sha1).digest()[:4] != buf[length - 4:]: + user_key = self.server_info.key + if hmac.new(user_key, buf[:-4], self.hashfunc).digest()[:4] != buf[-4:]: return b'' - return data + return buf[:-4] def server_udp_pre_encrypt(self, buf): - return buf + hmac.new(self.server_info.key, buf, hashlib.sha1).digest()[:4] + user_key = self.server_info.key + return buf + hmac.new(user_key, buf, self.hashfunc).digest()[:4] def server_udp_post_decrypt(self, buf): - length = len(buf) - data = buf[:-4] - if hmac.new(self.server_info.key, data, hashlib.sha1).digest()[:4] != buf[length - 4:]: + uid = buf[-8:-4] + user_key = self.server_info.key + if hmac.new(user_key, buf[:-4], self.hashfunc).digest()[:4] != buf[-4:]: return b'' - return data + return buf[:-8] From 293a937501e0d982af9bedf5c098947b5f5b86d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 3 Oct 2016 14:04:48 +0800 Subject: [PATCH 291/487] wait extra data --- shadowsocks/obfsplugin/auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index f96ab3ba..3d82c938 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1485,6 +1485,9 @@ def server_post_decrypt(self, buf): mac_key = self.server_info.recv_iv + self.server_info.key sha1data = hmac.new(mac_key, self.recv_buf[:3], self.hashfunc).digest()[:4] if sha1data != self.recv_buf[3:7]: + if self.method == self.no_compatible_method: + if len(self.recv_buf) < 31 + self.extra_wait_size: + return (b'', False) return self.not_match_return(self.recv_buf) if len(self.recv_buf) < 31: From 1183d567c931245536e0028892b82b1c885d505d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 4 Oct 2016 20:01:07 +0800 Subject: [PATCH 292/487] server can limit the host setting --- shadowsocks/obfsplugin/http_simple.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 8701bb45..3dfc071f 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -153,6 +153,14 @@ def get_data_from_http_header(self, buf): return ret_buf return b'' + def get_host_from_http_header(self, buf): + ret_buf = b'' + lines = buf.split(b'\r\n') + if lines and len(lines) > 4: + for line in lines: + if match_begin(line, b"Host: "): + return line[6:] + def not_match_return(self, buf): self.has_sent_header = True self.has_recv_header = True @@ -182,6 +190,14 @@ def server_decode(self, buf): if b'\r\n\r\n' in buf: datas = buf.split(b'\r\n\r\n', 1) ret_buf = self.get_data_from_http_header(buf) + host = self.get_host_from_http_header(buf) + if host and self.server_info.obfs_param: + pos = host.find(":") + if pos >= 0: + host = host[:pos] + hosts = self.server_info.obfs_param.split(',') + if host not in hosts: + return self.not_match_return(buf) if len(datas) > 1: ret_buf += datas[1] if len(ret_buf) >= 7: From d1f64de6b4003db7b2b9a5166ec5a47c397fe6a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 8 Oct 2016 11:21:30 +0800 Subject: [PATCH 293/487] -P => -O --- shadowsocks/shell.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 64fbef2a..44bdb630 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -136,11 +136,11 @@ def get_config(is_local): logging.basicConfig(level=logging.INFO, format='%(levelname)-s: %(message)s') if is_local: - shortopts = 'hd:s:b:p:k:l:m:P:o:G:g:c:t:vq' + shortopts = 'hd:s:b:p:k:l:m:O:o:G:g:c:t:vq' longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'user=', 'version'] else: - shortopts = 'hd:s:p:k:m:P:o:G:g:c:t:vq' + shortopts = 'hd:s:p:k:m:O:o:G:g:c:t:vq' longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'workers=', 'forbidden-ip=', 'user=', 'manager-address=', 'version'] try: @@ -174,7 +174,7 @@ def get_config(is_local): config['server'] = to_str(value) elif key == '-m': config['method'] = to_str(value) - elif key == '-P': + elif key == '-O': config['protocol'] = to_str(value) elif key == '-o': config['obfs'] = to_str(value) From c48190fe7a01738fd9b26c668ec5e7313be7598e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 11 Oct 2016 15:16:08 +0800 Subject: [PATCH 294/487] sort key, refine log --- mujson_mgr.py | 2 +- shadowsocks/obfsplugin/auth.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index d7a810a4..bbc8d3f5 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -64,7 +64,7 @@ def ssrlink(self, user, encode): def userinfo(self, user): ret = "" - for key in user.keys(): + for key in sorted(user): if key in ['enable']: continue ret += '\n' diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 3d82c938..a7200658 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -141,13 +141,13 @@ def insert(self, connection_id): self.re_enable(connection_id) self.update() if connection_id < self.front: - logging.warn('obfs auth: duplicate id') + logging.warn('obfs auth: deprecated id, someone replay attack') return False if connection_id > self.front + 0x4000: logging.warn('obfs auth: wrong id') return False if connection_id in self.alloc: - logging.warn('obfs auth: duplicate id 2') + logging.warn('obfs auth: duplicate id, someone replay attack') return False if self.back <= connection_id: self.back = connection_id + 1 From 45e27f914235c24fde1dac9480be4ef53a57c2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 11 Oct 2016 15:41:02 +0800 Subject: [PATCH 295/487] display version --- CHANGES | 3 +++ db_transfer.py | 1 + shadowsocks/server.py | 2 ++ shadowsocks/shell.py | 25 +++++++++++++++++++++---- version.py | 3 +++ 5 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 version.py diff --git a/CHANGES b/CHANGES index a58348a4..5ead42cf 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.9.4 2016-10-11 +- sync client version + 2.6.13 2015-11-02 - add protocol setting diff --git a/db_transfer.py b/db_transfer.py index de0a481d..46fbdc67 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -197,6 +197,7 @@ def thread_db(obj): last_rows = [] db_instance = obj() ServerPool.get_instance() + shell.log_shadowsocks_version() import resource logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) try: diff --git a/shadowsocks/server.py b/shadowsocks/server.py index f8deeeb6..7c60af95 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -37,6 +37,8 @@ def main(): config = shell.get_config(False) + shell.log_shadowsocks_version() + daemon.daemon_exec(config) if config['port_password']: diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 44bdb630..5973513c 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -54,13 +54,30 @@ def print_exception(e): def print_shadowsocks(): - version = '' + version_str = '' try: import pkg_resources - version = pkg_resources.get_distribution('shadowsocks').version + version_str = pkg_resources.get_distribution('shadowsocks').version except Exception: - pass - print('Shadowsocks %s' % version) + try: + from shadowsocks import version + version_str = version.version() + except Exception: + pass + print('ShadowsocksR %s' % version_str) + +def log_shadowsocks_version(): + version_str = '' + try: + import pkg_resources + version_str = pkg_resources.get_distribution('shadowsocks').version + except Exception: + try: + from shadowsocks import version + version_str = version.version() + except Exception: + pass + logging.info('ShadowsocksR %s' % version_str) def find_config(): config_path = 'user-config.json' diff --git a/version.py b/version.py new file mode 100644 index 00000000..244b7128 --- /dev/null +++ b/version.py @@ -0,0 +1,3 @@ +def version() + return '2.9.4 update 2016-10-11' + From 249b7fd73f6469ee7f7994733ca216ba12bb55fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 11 Oct 2016 15:47:07 +0800 Subject: [PATCH 296/487] fix path --- version.py => shadowsocks/version.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename version.py => shadowsocks/version.py (100%) diff --git a/version.py b/shadowsocks/version.py similarity index 100% rename from version.py rename to shadowsocks/version.py From 5345c66abe4bcad51f436adfaa207a7b4b9b6308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 11 Oct 2016 16:05:03 +0800 Subject: [PATCH 297/487] add License --- server_pool.py | 1 - shadowsocks/version.py | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/server_pool.py b/server_pool.py index ee0cad26..1fc23673 100644 --- a/server_pool.py +++ b/server_pool.py @@ -45,7 +45,6 @@ class ServerPool(object): def __init__(self): shell.check_python() self.config = shell.get_config(False) - shell.print_shadowsocks() self.dns_resolver = asyncdns.DNSResolver() if not self.config.get('dns_ipv6', False): asyncdns.IPV6_CONNECTION_SUPPORT = False diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 244b7128..be927589 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -1,3 +1,20 @@ -def version() +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright 2016 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +def version(): return '2.9.4 update 2016-10-11' From b8afd216d6f9960f2c22403b1d26484b292c7162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 12 Oct 2016 15:36:30 +0800 Subject: [PATCH 298/487] bump 2.9.5 update auth_aes128_md5 & auth_aes128_sha1 --- CHANGES | 3 +++ shadowsocks/obfsplugin/auth.py | 8 ++++---- shadowsocks/tcprelay.py | 2 ++ shadowsocks/udprelay.py | 1 + shadowsocks/version.py | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 5ead42cf..aad5b9a4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.9.5 2016-10-13 +- add auth_aes128_md5 and auth_aes128_sha1 + 2.9.4 2016-10-11 - sync client version diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index a7200658..dad22aa3 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1395,8 +1395,8 @@ def pack_auth_data(self, auth_data, buf): encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(self.server_info.key)) + self.salt, 'aes-128-cbc', b'\x00' * 16) data = uid + encryptor.encrypt(data)[16:] data += hmac.new(mac_key, data, self.hashfunc).digest()[:4] - check_head = os.urandom(3) - check_head += hmac.new(mac_key, check_head, self.hashfunc).digest()[:4] + check_head = os.urandom(1) + check_head += hmac.new(mac_key, check_head, self.hashfunc).digest()[:6] data = check_head + data + os.urandom(rnd_len) + buf data += hmac.new(self.user_key, data, self.hashfunc).digest()[:4] return data @@ -1483,8 +1483,8 @@ def server_post_decrypt(self, buf): if len(self.recv_buf) < 7: return (b'', False) mac_key = self.server_info.recv_iv + self.server_info.key - sha1data = hmac.new(mac_key, self.recv_buf[:3], self.hashfunc).digest()[:4] - if sha1data != self.recv_buf[3:7]: + sha1data = hmac.new(mac_key, self.recv_buf[:1], self.hashfunc).digest()[:6] + if sha1data != self.recv_buf[1:7]: if self.method == self.no_compatible_method: if len(self.recv_buf) < 31 + self.extra_wait_size: return (b'', False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 271e47a3..19a4af8b 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -129,6 +129,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.obfs_param = config['obfs_param'] server_info.iv = self._encryptor.cipher_iv server_info.recv_iv = b'' + server_info.key_str = common.to_bytes(config['password']) server_info.key = self._encryptor.cipher_key server_info.head_len = 30 server_info.tcp_mss = 1460 @@ -144,6 +145,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.obfs_param = '' server_info.iv = self._encryptor.cipher_iv server_info.recv_iv = b'' + server_info.key_str = common.to_bytes(config['password']) server_info.key = self._encryptor.cipher_key server_info.head_len = 30 server_info.tcp_mss = 1460 diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 6f595277..5e49ff41 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -907,6 +907,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun server_info.obfs_param = '' server_info.iv = b'' server_info.recv_iv = b'' + server_info.key_str = common.to_bytes(config['password']) server_info.key = encrypt.encrypt_key(self._password, self._method) server_info.head_len = 30 server_info.tcp_mss = 1440 diff --git a/shadowsocks/version.py b/shadowsocks/version.py index be927589..73fcd125 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '2.9.4 update 2016-10-11' + return '2.9.5 2016-10-13' From a0ee80a5ad1ee2da4c9c4d330b892495582d565b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 14 Oct 2016 17:53:20 +0800 Subject: [PATCH 299/487] new default config --- config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index f30ce55c..572bf580 100644 --- a/config.json +++ b/config.json @@ -8,9 +8,9 @@ "timeout": 120, "udp_timeout": 60, "method": "aes-256-cfb", - "protocol": "auth_sha1_v2_compatible", + "protocol": "auth_sha1_v4_compatible", "protocol_param": "", - "obfs": "http_simple_compatible", + "obfs": "tls1.2_ticket_auth_compatible", "obfs_param": "", "dns_ipv6": false, "connect_verbose_info": 0, From 685b62532eb52586d9c805a3d981cc3636983a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 16 Oct 2016 21:55:06 +0800 Subject: [PATCH 300/487] 2.9.5.1 UDP bind address (set out_bind and out_bindv6) UDP over TCP auto bind return a long enough err string --- shadowsocks/obfsplugin/auth.py | 32 ++++++++++----------- shadowsocks/obfsplugin/http_simple.py | 6 ++-- shadowsocks/obfsplugin/obfs_tls.py | 2 +- shadowsocks/obfsplugin/verify.py | 8 +++--- shadowsocks/tcprelay.py | 40 +++++++++++++++------------ shadowsocks/udprelay.py | 23 ++++++++++++++- shadowsocks/version.py | 2 +- 7 files changed, 70 insertions(+), 43 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index dad22aa3..19e87556 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -110,7 +110,7 @@ def server_decode(self, buf): def not_match_return(self, buf): self.raw_trans = True if self.method == self.no_compatible_method: - return (b'E'*64, False) + return (b'E'*2048, False) return (buf, False) class client_queue(object): @@ -367,7 +367,7 @@ def server_post_decrypt(self, buf): self.recv_buf = b'' if self.decrypt_packet_num == 0: logging.info('auth_sha1: over size') - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -378,7 +378,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data uncorrect checksum') @@ -601,7 +601,7 @@ def server_post_decrypt(self, buf): self.recv_buf = b'' if self.decrypt_packet_num == 0: logging.info('auth_sha1_v2: over size') - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -612,7 +612,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data uncorrect checksum') @@ -810,7 +810,7 @@ def server_post_decrypt(self, buf): self.recv_buf = b'' if self.decrypt_packet_num == 0: logging.info('auth_sha1_v3: over size') - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -821,7 +821,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data uncorrect checksum') @@ -1027,7 +1027,7 @@ def server_post_decrypt(self, buf): logging.info('auth_sha1_v4: wrong crc') if self.decrypt_packet_num == 0: logging.info('auth_sha1_v4: wrong crc') - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') length = struct.unpack('>H', self.recv_buf[:2])[0] @@ -1036,7 +1036,7 @@ def server_post_decrypt(self, buf): self.recv_buf = b'' if self.decrypt_packet_num == 0: logging.info('auth_sha1_v4: over size') - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -1047,7 +1047,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data uncorrect checksum') @@ -1263,7 +1263,7 @@ def server_post_decrypt(self, buf): logging.info('auth_aes128: wrong crc') if self.recv_id == 0: logging.info('auth_aes128: wrong crc') - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') length = struct.unpack(' len(self.recv_buf): @@ -1283,7 +1283,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.recv_id == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data uncorrect checksum') @@ -1539,7 +1539,7 @@ def server_post_decrypt(self, buf): logging.info(self.no_compatible_method + ': wrong crc') if self.recv_id == 0: logging.info(self.no_compatible_method + ': wrong crc') - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') length = struct.unpack(' len(self.recv_buf): @@ -1559,7 +1559,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.recv_id == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data uncorrect checksum') diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 3dfc071f..37831ebc 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -165,7 +165,7 @@ def not_match_return(self, buf): self.has_sent_header = True self.has_recv_header = True if self.method == 'http_simple': - return (b'E'*64, False, False) + return (b'E'*2048, False, False) return (buf, True, False) def server_decode(self, buf): @@ -251,7 +251,7 @@ def not_match_return(self, buf): self.has_sent_header = True self.has_recv_header = True if self.method == 'http_post': - return (b'E'*64, False, False) + return (b'E'*2048, False, False) return (buf, True, False) class random_head(plain.plain): @@ -300,7 +300,7 @@ def server_decode(self, buf): if crc != 0xffffffff: self.has_sent_header = True if self.method == 'random_head': - return (b'E'*64, False, False) + return (b'E'*2048, False, False) return (buf, True, False) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index cc7183e1..9912c92f 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -167,7 +167,7 @@ def server_encode(self, buf): def decode_error_return(self, buf): self.handshake_status = -1 if self.method == 'tls1.2_ticket_auth': - return (b'E'*64, False, False) + return (b'E'*2048, False, False) return (buf, True, False) def server_decode(self, buf): diff --git a/shadowsocks/obfsplugin/verify.py b/shadowsocks/obfsplugin/verify.py index 674652f9..e896fba6 100644 --- a/shadowsocks/obfsplugin/verify.py +++ b/shadowsocks/obfsplugin/verify.py @@ -160,7 +160,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -170,7 +170,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return (b'E', False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data uncorrect CRC32') @@ -245,7 +245,7 @@ def server_post_decrypt(self, buf): self.raw_trans = True self.recv_buf = b'' if self.decrypt_packet_num == 0: - return (b'E'*64, False) + return (b'E'*2048, False) else: raise Exception('server_post_decrype data error') if length > len(self.recv_buf): @@ -304,7 +304,7 @@ def server_pre_encrypt(self, buf): def not_match_return(self, buf): self.raw_trans = True if self.method == 'verify_sha1': - return (b'E'*64, False) + return (b'E'*2048, False) return (buf, False) def server_post_decrypt(self, buf): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 19a4af8b..d65833ca 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -532,6 +532,24 @@ def _handle_stage_addr(self, ogn_data, data): traceback.print_exc() self.destroy() + def _socket_bind_addr(self, sock, af): + bind_addr = '' + if self._bind and af == socket.AF_INET: + bind_addr = self._bind + elif self._bindv6 and af == socket.AF_INET6: + bind_addr = self._bindv6 + else: + bind_addr = self._accept_address[0] + + bind_addr = bind_addr.replace("::ffff:", "") + if bind_addr in self._ignore_bind_list: + bind_addr = None + if bind_addr: + local_addrs = socket.getaddrinfo(bind_addr, 0, 0, socket.SOCK_STREAM, socket.SOL_TCP) + if local_addrs[0][0] == af: + logging.debug("bind %s" % (bind_addr,)) + sock.bind((bind_addr, 0)) + def _create_remote_socket(self, ip, port): if self._remote_udp: addrs_v6 = socket.getaddrinfo("::", 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) @@ -568,26 +586,14 @@ def _create_remote_socket(self, ip, port): remote_sock.setblocking(False) if self._remote_udp: remote_sock_v6.setblocking(False) + + if not self._is_local: + self._socket_bind_addr(remote_sock, af) + self._socket_bind_addr(remote_sock_v6, af) else: remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - if not self._is_local: - bind_addr = '' - if self._bind and af == socket.AF_INET: - bind_addr = self._bind - elif self._bindv6 and af == socket.AF_INET6: - bind_addr = self._bindv6 - else: - bind_addr = self._accept_address[0] - - bind_addr = bind_addr.replace("::ffff:", "") - if bind_addr in self._ignore_bind_list: - bind_addr = None - if bind_addr: - local_addrs = socket.getaddrinfo(bind_addr, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) - if local_addrs[0][0] == af: - logging.debug("bind %s" % (bind_addr,)) - remote_sock.bind((bind_addr, 0)) + self._socket_bind_addr(remote_sock, af) return remote_sock def _handle_dns_resolved(self, result, error): diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 5e49ff41..30649a6b 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -923,6 +923,10 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self._timeout_offset = 0 # last checked position for timeout self._handler_to_timeouts = {} # key: handler value: index in timeouts + self._bind = config.get('out_bind', '') + self._bindv6 = config.get('out_bindv6', '') + self._ignore_bind_list = config.get('ignore_bind', []) + if 'forbidden_ip' in config: self._forbidden_iplist = config['forbidden_ip'] else: @@ -1010,6 +1014,22 @@ def _handel_protocol_error(self, client_address, ogn_data): #raise Exception('can not parse header') logging.warn("Protocol ERROR, UDP ogn data %s from %s:%d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1])) + def _socket_bind_addr(self, sock, af): + bind_addr = '' + if self._bind and af == socket.AF_INET: + bind_addr = self._bind + elif self._bindv6 and af == socket.AF_INET6: + bind_addr = self._bindv6 + + bind_addr = bind_addr.replace("::ffff:", "") + if bind_addr in self._ignore_bind_list: + bind_addr = None + if bind_addr: + local_addrs = socket.getaddrinfo(bind_addr, 0, 0, socket.SOCK_STREAM, socket.SOL_TCP) + if local_addrs[0][0] == af: + logging.debug("bind %s" % (bind_addr,)) + sock.bind((bind_addr, 0)) + def _handle_server(self): server = self._server_socket data, r_addr = server.recvfrom(BUF_SIZE) @@ -1045,7 +1065,7 @@ def _handle_server(self): if type(data) is tuple: return - return self._handle_tcp_over_udp(data, r_addr) + #return self._handle_tcp_over_udp(data, r_addr) try: header_result = parse_header(data) @@ -1094,6 +1114,7 @@ def _handle_server(self): return client = socket.socket(af, socktype, proto) client.setblocking(False) + self._socket_bind_addr(client, af) is_dns = False if len(data) > 20 and data[11:19] == b"\x00\x01\x00\x00\x00\x00\x00\x00": is_dns = True diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 73fcd125..e7837559 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '2.9.5 2016-10-13' + return '2.9.5.1 2016-10-16' From cfb894400321afe362946a0056d65317fc2c2283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 17 Oct 2016 16:02:11 +0800 Subject: [PATCH 301/487] tls1.2_ticket_auth random packet size --- CHANGES | 6 ++++++ shadowsocks/obfsplugin/obfs_tls.py | 9 +++++---- shadowsocks/version.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index aad5b9a4..a00c92f0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +2.9.5.1 2016-10-17 +- tls1.2_ticket_auth random packet size + +2.9.5.1 2016-10-16 +- UDP bind address + 2.9.5 2016-10-13 - add auth_aes128_md5 and auth_aes128_sha1 diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 9912c92f..a8526bfa 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -149,9 +149,10 @@ def server_encode(self, buf): return buf if self.handshake_status == 8: ret = b'' - while len(buf) > 8192: - ret += b"\x17" + self.tls_version + struct.pack('>H', 8192) + buf[:8192] - buf = buf[8192:] + while len(buf) > 4096: + size = struct.unpack('>H', os.urandom(2))[0] % 4096 + 100 + ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size] + buf = buf[size:] if len(buf) > 0: ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf return ret @@ -178,7 +179,7 @@ def server_decode(self, buf): ret = b'' self.recv_buffer += buf while len(self.recv_buffer) > 5: - if ord(self.recv_buffer[0]) != 0x17: + if ord(self.recv_buffer[0]) != 0x17 or ord(self.recv_buffer[1]) != 0x3 or ord(self.recv_buffer[2]) != 0x3: logging.info("data = %s" % (binascii.hexlify(self.recv_buffer))) raise Exception('server_decode appdata error') size = struct.unpack('>H', self.recv_buffer[3:5])[0] diff --git a/shadowsocks/version.py b/shadowsocks/version.py index e7837559..a2abded5 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '2.9.5.1 2016-10-16' + return '2.9.6 2016-10-17' From 2eaca2faeaa2fb29d079c06507813d0d8c03d514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 17 Oct 2016 16:07:25 +0800 Subject: [PATCH 302/487] fix typo --- shadowsocks/obfsplugin/obfs_tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index a8526bfa..691e6fa9 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -149,7 +149,7 @@ def server_encode(self, buf): return buf if self.handshake_status == 8: ret = b'' - while len(buf) > 4096: + while len(buf) > 4196: size = struct.unpack('>H', os.urandom(2))[0] % 4096 + 100 ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size] buf = buf[size:] From 13ffdac2b76697f819ca2b5e21cb78079601fc66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 18 Oct 2016 13:36:22 +0800 Subject: [PATCH 303/487] tls1.2 split pack data --- CHANGES | 2 +- shadowsocks/obfsplugin/obfs_tls.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index a00c92f0..116c222c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,4 @@ -2.9.5.1 2016-10-17 +2.9.6 2016-10-17 - tls1.2_ticket_auth random packet size 2.9.5.1 2016-10-16 diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 691e6fa9..46ecc9ba 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -84,7 +84,14 @@ def client_encode(self, buf): if self.handshake_status == -1: return buf if self.handshake_status == 8: - return b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf + ret = b'' + while len(buf) > 2048: + size = min(struct.unpack('>H', os.urandom(2))[0] % 4096 + 100, len(buf)) + ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size] + buf = buf[size:] + if len(buf) > 0: + ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf + return ret self.send_buffer += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf if self.handshake_status == 0: self.handshake_status = 1 @@ -149,8 +156,8 @@ def server_encode(self, buf): return buf if self.handshake_status == 8: ret = b'' - while len(buf) > 4196: - size = struct.unpack('>H', os.urandom(2))[0] % 4096 + 100 + while len(buf) > 2048: + size = min(struct.unpack('>H', os.urandom(2))[0] % 4096 + 100, len(buf)) ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size] buf = buf[size:] if len(buf) > 0: From e5824be14579c8d8d32b6814fbbf03eddf6abef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 25 Oct 2016 00:45:19 +0800 Subject: [PATCH 304/487] fix none encrypt key size --- shadowsocks/crypto/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/table.py b/shadowsocks/crypto/table.py index c5d613f4..fd6aa94c 100644 --- a/shadowsocks/crypto/table.py +++ b/shadowsocks/crypto/table.py @@ -73,7 +73,7 @@ def update(self, data): return data ciphers = { - 'none': (0, 0, NoneCipher), + 'none': (16, 0, NoneCipher), 'table': (0, 0, TableCipher) } From c7fb9bcb5e9b3d28dd11dc88bbee35bb314db494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 28 Oct 2016 18:17:28 +0800 Subject: [PATCH 305/487] init script & update README.md --- README.md | 57 +++++++++++++++++++++++++++--------------------------- initcfg.sh | 7 +++++++ 2 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 initcfg.sh diff --git a/README.md b/README.md index 9c38e3e4..b6b199f9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -shadowsocks +ShadowsocksR =========== -[![PyPI version]][PyPI] [![Build Status]][Travis CI] -[![Coverage Status]][Coverage] A fast tunnel proxy that helps you bypass firewalls. @@ -14,36 +12,47 @@ Server Debian / Ubuntu: - apt-get install python-pip - pip install shadowsocks + apt-get install git + git clone https://github.com/breakwa11/shadowsocks.git CentOS: - yum install python-setuptools && easy_install pip - pip install shadowsocks + yum install git + git clone https://github.com/breakwa11/shadowsocks.git Windows: -See [Install Server on Windows] + git clone https://github.com/breakwa11/shadowsocks.git -### Usage +### Usage for single user on linux platform - ssserver -p 443 -k password -m aes-256-cfb +If you clone it into "~/shadowsocks" +move to "~/shadowsocks", then run: + + bash initcfg.sh + +move to "~/shadowsocks/shadowsocks", then run: + + python server.py -p 443 -k password -m aes-128-cfb -O auth_aes128_md5_compatible -o tls1.2_ticket_auth_compatible + +Check all the options via `-h`. + +You can also use a configuration file instead (recommand), move to "~/shadowsocks" and edit the file "user-config.json", then move to "~/shadowsocks/shadowsocks" again, just run: + + python server.py To run in the background: - sudo ssserver -p 443 -k password -m aes-256-cfb --user nobody -d start + ./logrun.sh To stop: - sudo ssserver -d stop + ./stop.sh -To check the log: +To monitor the log: - sudo less /var/log/shadowsocks.log + ./tail.sh -Check all the options via `-h`. You can also use a [Configuration] file -instead. Client ------ @@ -80,27 +89,17 @@ under the License. Bugs and Issues ---------------- -* [Troubleshooting] * [Issue Tracker] -* [Mailing list] [Android]: https://github.com/shadowsocks/shadowsocks-android [Build Status]: https://travis-ci.org/falseen/shadowsocks.svg?branch=manyuser-travis -[Configuration]: https://github.com/shadowsocks/shadowsocks/wiki/Configuration-via-Config-File -[Coverage Status]: https://jenkins.shadowvpn.org/result/shadowsocks -[Coverage]: https://jenkins.shadowvpn.org/job/Shadowsocks/ws/PYENV/py34/label/linux/htmlcov/index.html [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help -[Issue Tracker]: https://github.com/shadowsocks/shadowsocks/issues?state=open -[Install Server on Windows]: https://github.com/shadowsocks/shadowsocks/wiki/Install-Shadowsocks-Server-on-Windows -[Mailing list]: https://groups.google.com/group/shadowsocks +[Issue Tracker]: https://github.com/breakwa11/shadowsocks/issues?state=open [OpenWRT]: https://github.com/shadowsocks/openwrt-shadowsocks [OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help -[PyPI]: https://pypi.python.org/pypi/shadowsocks -[PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat [Travis CI]: https://travis-ci.org/falseen/shadowsocks -[Troubleshooting]: https://github.com/shadowsocks/shadowsocks/wiki/Troubleshooting -[Wiki]: https://github.com/shadowsocks/shadowsocks/wiki -[Windows]: https://github.com/shadowsocks/shadowsocks-csharp +[Windows]: https://github.com/breakwa11/shadowsocks-csharp +[Wiki]: https://github.com/breakwa11/shadowsocks-rss/wiki \ No newline at end of file diff --git a/initcfg.sh b/initcfg.sh new file mode 100644 index 00000000..90f72082 --- /dev/null +++ b/initcfg.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +chmod +x *.sh +chmod +x shadowsocks/*.sh +cp -n apiconfig.py userapiconfig.py +cp -n config.json user-config.json + From 9533044bd8f3b540df5917663eb8a5910820d6a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 28 Oct 2016 19:18:38 +0800 Subject: [PATCH 306/487] update init script --- initcfg.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/initcfg.sh b/initcfg.sh index 90f72082..862d1abe 100644 --- a/initcfg.sh +++ b/initcfg.sh @@ -4,4 +4,5 @@ chmod +x *.sh chmod +x shadowsocks/*.sh cp -n apiconfig.py userapiconfig.py cp -n config.json user-config.json +cp -n mysql.json usermysql.json From ac580205959008232eb1255b7512b2ff37b0d9f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 2 Nov 2016 19:06:54 +0800 Subject: [PATCH 307/487] print RLIMIT_NOFILE always close cymysql objects --- db_transfer.py | 65 ++++++++++++++++++++++++++++++++----------- shadowsocks/server.py | 3 ++ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 46fbdc67..4656ef00 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -301,10 +301,20 @@ def update_all_user(self, dt_transfer): user=self.cfg["user"], passwd=self.cfg["password"], db=self.cfg["db"], charset='utf8') - cur = conn.cursor() - cur.execute(query_sql) - cur.close() - conn.commit() + try: + cur = conn.cursor() + try: + cur.execute(query_sql) + except Exception as e: + logging.error(e) + update_transfer = {} + + cur.close() + conn.commit() + except Exception as e: + logging.error(e) + update_transfer = {} + conn.close() return update_transfer @@ -321,8 +331,13 @@ def pull_db_all_user(self): user=self.cfg["user"], passwd=self.cfg["password"], db=self.cfg["db"], charset='utf8') - rows = self.pull_db_users(conn) + try: + rows = self.pull_db_users(conn) + except Exception as e: + logging.error(e) + rows = [] conn.close() + if not rows: logging.warn('no user in db') return rows @@ -414,19 +429,28 @@ def update_all_user(self, dt_transfer): ' END, t = ' + str(int(last_time)) + \ ' WHERE port IN (%s)' % query_sub_in cur = conn.cursor() - cur.execute(query_sql) + try: + cur.execute(query_sql) + except Exception as e: + logging.error(e) cur.close() try: cur = conn.cursor() - cur.execute("INSERT INTO `ss_node_online_log` (`id`, `node_id`, `online_user`, `log_time`) VALUES (NULL, '" + \ + try: + cur.execute("INSERT INTO `ss_node_online_log` (`id`, `node_id`, `online_user`, `log_time`) VALUES (NULL, '" + \ str(self.cfg["node_id"]) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") + except Exception as e: + logging.error(e) cur.close() cur = conn.cursor() - cur.execute("INSERT INTO `ss_node_info_log` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ + try: + cur.execute("INSERT INTO `ss_node_info_log` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ str(self.cfg["node_id"]) + "', '" + str(self.uptime()) + "', '" + \ str(self.load()) + "', unix_timestamp()); ") + except Exception as e: + logging.error(e) cur.close() except: logging.warn('no `ss_node_online_log` or `ss_node_info_log` in db') @@ -444,8 +468,12 @@ def pull_db_users(self, conn): cur = conn.cursor() node_info_keys = ['traffic_rate'] - cur.execute("SELECT " + ','.join(node_info_keys) +" FROM ss_node where `id`='" + str(self.cfg["node_id"]) + "'") - nodeinfo = cur.fetchone() + try: + cur.execute("SELECT " + ','.join(node_info_keys) +" FROM ss_node where `id`='" + str(self.cfg["node_id"]) + "'") + nodeinfo = cur.fetchone() + except Exception as e: + logging.error(e) + nodeinfo = None if nodeinfo == None: rows = [] @@ -461,13 +489,16 @@ def pull_db_users(self, conn): self.cfg['transfer_mul'] = float(node_info_dict['traffic_rate']) cur = conn.cursor() - cur.execute("SELECT " + ','.join(keys) + " FROM user") - rows = [] - for r in cur.fetchall(): - d = {} - for column in range(len(keys)): - d[keys[column]] = r[column] - rows.append(d) + try: + cur.execute("SELECT " + ','.join(keys) + " FROM user") + rows = [] + for r in cur.fetchall(): + d = {} + for column in range(len(keys)): + d[keys[column]] = r[column] + rows.append(d) + except Exception as e: + logging.error(e) cur.close() return rows diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 7c60af95..6ddd5909 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -41,6 +41,9 @@ def main(): daemon.daemon_exec(config) + import resource + logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) + if config['port_password']: pass else: From 2a6fbc87d48858eaab3fdc7e6f632ecbd8701047 Mon Sep 17 00:00:00 2001 From: "V.E.O" Date: Thu, 10 Nov 2016 11:25:12 +0800 Subject: [PATCH 308/487] fix import error of resource on windows --- shadowsocks/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 6ddd5909..aa67303a 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -41,8 +41,11 @@ def main(): daemon.daemon_exec(config) - import resource - logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) + try: + import resource + logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) + except ImportError: + pass if config['port_password']: pass From ce25bbf650107441be067146840688e79b97e730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 22 Nov 2016 07:23:30 +0800 Subject: [PATCH 309/487] catch bind error --- shadowsocks/tcprelay.py | 5 ++++- shadowsocks/udprelay.py | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index d65833ca..441931d1 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -548,7 +548,10 @@ def _socket_bind_addr(self, sock, af): local_addrs = socket.getaddrinfo(bind_addr, 0, 0, socket.SOCK_STREAM, socket.SOL_TCP) if local_addrs[0][0] == af: logging.debug("bind %s" % (bind_addr,)) - sock.bind((bind_addr, 0)) + try: + sock.bind((bind_addr, 0)) + except Exception as e: + logging.warn("bind %s fail" % (bind_addr,)) def _create_remote_socket(self, ip, port): if self._remote_udp: diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 30649a6b..78b915d2 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1025,10 +1025,13 @@ def _socket_bind_addr(self, sock, af): if bind_addr in self._ignore_bind_list: bind_addr = None if bind_addr: - local_addrs = socket.getaddrinfo(bind_addr, 0, 0, socket.SOCK_STREAM, socket.SOL_TCP) + local_addrs = socket.getaddrinfo(bind_addr, 0, 0, socket.SOCK_DGRAM, socket.SOL_UDP) if local_addrs[0][0] == af: logging.debug("bind %s" % (bind_addr,)) - sock.bind((bind_addr, 0)) + try: + sock.bind((bind_addr, 0)) + except Exception as e: + logging.warn("bind %s fail" % (bind_addr,)) def _handle_server(self): server = self._server_socket From c3a96129196ef15b14c41e4ff3e5c7916f567bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 22 Nov 2016 15:27:12 +0800 Subject: [PATCH 310/487] bump 2.9.7 manage client with LRUCache --- CHANGES | 8 +++++ shadowsocks/lru_cache.py | 7 ++-- shadowsocks/tcprelay.py | 74 +++++++++----------------------------- shadowsocks/udprelay.py | 76 ++++++++++------------------------------ shadowsocks/version.py | 2 +- 5 files changed, 49 insertions(+), 118 deletions(-) diff --git a/CHANGES b/CHANGES index 116c222c..e865e964 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +2.9.7 2016-11-22 +- manage client with LRUCache +- catch bind error +- fix import error of resource on windows +- print RLIMIT_NOFILE +- always close cymysql objects +- add init script + 2.9.6 2016-10-17 - tls1.2_ticket_auth random packet size diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 9017f81c..d748d587 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -74,6 +74,9 @@ def __delitem__(self, key): del self._store[key] del self._keys_to_last_time[key] + def __contains__(self, key): + return key in self._store + def __iter__(self): return iter(self._store) @@ -98,10 +101,10 @@ def sweep(self): if now - last_t <= self.timeout: break value = self._store[key] - if self.close_callback is not None: - self.close_callback(value) del self._store[key] del self._keys_to_last_time[key] + if self.close_callback is not None: + self.close_callback(value) c += 1 if c: logging.debug('%d keys swept' % c) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 441931d1..66783016 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -28,7 +28,7 @@ import random import platform -from shadowsocks import encrypt, obfs, eventloop, shell, common +from shadowsocks import encrypt, obfs, eventloop, shell, common, lru_cache from shadowsocks.common import pre_parse_header, parse_header # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time @@ -961,10 +961,8 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun common.connect_log = logging.info self._timeout = config['timeout'] - self._timeouts = [] # a list for all the handlers - # we trim the timeouts once a while - self._timeout_offset = 0 # last checked position for timeout - self._handler_to_timeouts = {} # key: handler value: index in timeouts + self._timeout_cache = lru_cache.LRUCache(timeout=self._timeout, + close_callback=self._close_tcp_client) if is_local: listen_addr = config['local_address'] @@ -1005,12 +1003,9 @@ def add_to_loop(self, loop): eventloop.POLL_IN | eventloop.POLL_ERR, self) self._eventloop.add_periodic(self.handle_periodic) - def remove_handler(self, handler): - index = self._handler_to_timeouts.get(hash(handler), -1) - if index >= 0: - # delete is O(n), so we just set it to None - self._timeouts[index] = None - del self._handler_to_timeouts[hash(handler)] + def remove_handler(self, client): + if hash(client) in self._timeout_cache: + del self._timeout_cache[hash(client)] def add_connection(self, val): self.server_connections += val @@ -1052,57 +1047,22 @@ def stat_add(self, local_addr, val): logging.info('Total connections down to %d' % newval) self._stat_counter[-1] = self._stat_counter.get(-1, 0) - connections_step - def update_activity(self, handler, data_len): + def update_activity(self, client, data_len): if data_len and self._stat_callback: self._stat_callback(self._listen_port, data_len) - # set handler to active - now = int(time.time()) - if now - handler.last_activity < eventloop.TIMEOUT_PRECISION: - # thus we can lower timeout modification frequency - return - handler.last_activity = now - index = self._handler_to_timeouts.get(hash(handler), -1) - if index >= 0: - # delete is O(n), so we just set it to None - self._timeouts[index] = None - length = len(self._timeouts) - self._timeouts.append(handler) - self._handler_to_timeouts[hash(handler)] = length + self._timeout_cache[hash(client)] = client def _sweep_timeout(self): - # tornado's timeout memory management is more flexible than we need - # we just need a sorted last_activity queue and it's faster than heapq - # in fact we can do O(1) insertion/remove so we invent our own - if self._timeouts: - logging.log(shell.VERBOSE_LEVEL, 'sweeping timeouts') - now = time.time() - length = len(self._timeouts) - pos = self._timeout_offset - while pos < length: - handler = self._timeouts[pos] - if handler: - if now - handler.last_activity < self._timeout: - break - else: - if handler.remote_address: - logging.debug('timed out: %s:%d' % - handler.remote_address) - else: - logging.debug('timed out') - handler.destroy() - self._timeouts[pos] = None # free memory - pos += 1 - else: - pos += 1 - if pos > TIMEOUTS_CLEAN_SIZE and pos > length >> 1: - # clean up the timeout queue when it gets larger than half - # of the queue - self._timeouts = self._timeouts[pos:] - for key in self._handler_to_timeouts: - self._handler_to_timeouts[key] -= pos - pos = 0 - self._timeout_offset = pos + self._timeout_cache.sweep() + + def _close_tcp_client(self, client): + if client.remote_address: + logging.debug('timed out: %s:%d' % + client.remote_address) + else: + logging.debug('timed out') + client.destroy() def handle_event(self, sock, fd, event): # handle events and dispatch to handlers diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 78b915d2..9d62127a 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -918,10 +918,8 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self._reqid_to_hd = {} self._data_to_write_to_server_socket = [] - self._timeouts = [] # a list for all the handlers - # we trim the timeouts once a while - self._timeout_offset = 0 # last checked position for timeout - self._handler_to_timeouts = {} # key: handler value: index in timeouts + self._timeout_cache = lru_cache.LRUCache(timeout=self._timeout, + close_callback=self._close_tcp_client) self._bind = config.get('out_bind', '') self._bindv6 = config.get('out_bindv6', '') @@ -1317,62 +1315,24 @@ def add_to_loop(self, loop): eventloop.POLL_IN | eventloop.POLL_ERR, self) loop.add_periodic(self.handle_periodic) - def remove_handler(self, handler): - index = self._handler_to_timeouts.get(hash(handler), -1) - if index >= 0: - # delete is O(n), so we just set it to None - self._timeouts[index] = None - del self._handler_to_timeouts[hash(handler)] - - def update_activity(self, handler): - # set handler to active - now = int(time.time()) - if now - handler.last_activity < eventloop.TIMEOUT_PRECISION: - # thus we can lower timeout modification frequency - return - handler.last_activity = now - index = self._handler_to_timeouts.get(hash(handler), -1) - if index >= 0: - # delete is O(n), so we just set it to None - self._timeouts[index] = None - length = len(self._timeouts) - self._timeouts.append(handler) - self._handler_to_timeouts[hash(handler)] = length + def remove_handler(self, client): + if hash(client) in self._timeout_cache: + del self._timeout_cache[hash(client)] + + def update_activity(self, client): + self._timeout_cache[hash(client)] = client def _sweep_timeout(self): - # tornado's timeout memory management is more flexible than we need - # we just need a sorted last_activity queue and it's faster than heapq - # in fact we can do O(1) insertion/remove so we invent our own - if self._timeouts: - logging.log(shell.VERBOSE_LEVEL, 'sweeping timeouts') - now = time.time() - length = len(self._timeouts) - pos = self._timeout_offset - while pos < length: - handler = self._timeouts[pos] - if handler: - if now - handler.last_activity < self._timeout: - break - else: - if handler.remote_address: - logging.debug('timed out: %s:%d' % - handler.remote_address) - else: - logging.debug('timed out') - handler.destroy() - handler.destroy_local() - self._timeouts[pos] = None # free memory - pos += 1 - else: - pos += 1 - if pos > TIMEOUTS_CLEAN_SIZE and pos > length >> 1: - # clean up the timeout queue when it gets larger than half - # of the queue - self._timeouts = self._timeouts[pos:] - for key in self._handler_to_timeouts: - self._handler_to_timeouts[key] -= pos - pos = 0 - self._timeout_offset = pos + self._timeout_cache.sweep() + + def _close_tcp_client(self, client): + if client.remote_address: + logging.debug('timed out: %s:%d' % + client.remote_address) + else: + logging.debug('timed out') + client.destroy() + client.destroy_local() def handle_event(self, sock, fd, event): if sock == self._server_socket: diff --git a/shadowsocks/version.py b/shadowsocks/version.py index a2abded5..d0befd04 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '2.9.6 2016-10-17' + return '2.9.7 2016-11-22' From d45ba8de03b4f152c492299b4e7708f8cc3a277e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 22 Nov 2016 23:10:32 +0800 Subject: [PATCH 311/487] OpenSSL 1.1.0 compatible --- shadowsocks/crypto/openssl.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shadowsocks/crypto/openssl.py b/shadowsocks/crypto/openssl.py index 630baefe..24264f18 100644 --- a/shadowsocks/crypto/openssl.py +++ b/shadowsocks/crypto/openssl.py @@ -49,7 +49,10 @@ def load_openssl(): libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p, c_char_p, c_int) - libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) + if hasattr(libcrypto, "EVP_CIPHER_CTX_cleanup"): + libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) + else: + libcrypto.EVP_CIPHER_CTX_reset.argtypes = (c_void_p,) libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,) libcrypto.RAND_bytes.restype = c_int @@ -120,7 +123,10 @@ def __del__(self): def clean(self): if self._ctx: - libcrypto.EVP_CIPHER_CTX_cleanup(self._ctx) + if hasattr(libcrypto, "EVP_CIPHER_CTX_cleanup"): + libcrypto.EVP_CIPHER_CTX_cleanup(self._ctx) + else: + libcrypto.EVP_CIPHER_CTX_reset(self._ctx) libcrypto.EVP_CIPHER_CTX_free(self._ctx) From e048e224d09baec5669a22a36d69ce130981db74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 24 Nov 2016 19:13:32 +0800 Subject: [PATCH 312/487] Do not return empty list if raise exception --- db_transfer.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 4656ef00..dcc158ea 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -314,8 +314,9 @@ def update_all_user(self, dt_transfer): except Exception as e: logging.error(e) update_transfer = {} + finally: + conn.close() - conn.close() return update_transfer def pull_db_all_user(self): @@ -333,10 +334,8 @@ def pull_db_all_user(self): try: rows = self.pull_db_users(conn) - except Exception as e: - logging.error(e) - rows = [] - conn.close() + finally: + conn.close() if not rows: logging.warn('no user in db') From bdee67b856767cc5038a0328a5feeaa9733e5f55 Mon Sep 17 00:00:00 2001 From: shoaly Date: Sun, 18 Dec 2016 20:20:41 +0800 Subject: [PATCH 313/487] create a Dockerfile for ssr (#143) by this dockerfile, make it easy to run ssr-server on docker server --- Dockerfile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a065b7a9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM alpine + +ENV SERVER_ADDR 0.0.0.0 +ENV SERVER_PORT 51348 +ENV PASSWORD= +ENV METHOD aes-256-cfb +ENV PROTOCOL origin +ENV OBFS http_simple +ENV TIMEOUT 300 +ENV DNS_ADDR 8.8.8.8 +ENV DNS_ADDR_2 8.8.4.4 + + +RUN apk update \ + && apk add python \ + libsodium \ + unzip \ + wget \ + && rm -rf /var/cache/apk/* + + + +RUN wget --no-check-certificate https://github.com/breakwa11/shadowsocks/archive/manyuser.zip -O /tmp/manyuser.zip \ + && unzip -d /tmp /tmp/manyuser.zip \ + && mv /tmp/shadowsocks-manyuser/shadowsocks ~/shadowsocks \ + && rm -rf /tmp/* + + +WORKDIR ~/shadowsocks + + +CMD python ~/shadowsocks/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS From e0b4bdf5516807fec53bd0adb84a87f4ec191b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 23 Dec 2016 20:58:43 +0800 Subject: [PATCH 314/487] refine obfs, support comment in config.json --- mujson_mgr.py | 8 +++-- shadowsocks/obfsplugin/http_simple.py | 4 +-- shadowsocks/obfsplugin/obfs_tls.py | 14 +++++---- shadowsocks/server.py | 2 +- shadowsocks/shell.py | 43 ++++++++++++++++++++++++++- 5 files changed, 60 insertions(+), 11 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index bbc8d3f5..179723e6 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -91,7 +91,7 @@ def rand_pass(self): return ''.join([random.choice('''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i in range(8)]) def add(self, user): - up = {'enable': True, 'u': 0, 'd': 0, 'method': "aes-128-cfb", + up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-cfb", 'protocol': "auth_sha1_v4_compatible", 'obfs': "tls1.2_ticket_auth_compatible", 'transfer_enable': 1125899906842624} @@ -217,7 +217,11 @@ def main(): '3': 'auth_sha1_v2', '+4': 'auth_sha1_v4_compatible', '4': 'auth_sha1_v4', - 'a1': 'auth_aes128'} + 'am': 'auth_aes128_md5', + 'as': 'auth_aes128_sha1', + '+am': 'auth_aes128_md5_compatible', + '+as': 'auth_aes128_sha1_compatible' + } fast_set_method = {'a0': 'aes-128-cfb', 'a1': 'aes-192-cfb', 'a2': 'aes-256-cfb', diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 37831ebc..596b6497 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -138,7 +138,7 @@ def server_encode(self, buf): def get_data_from_http_header(self, buf): ret_buf = b'' lines = buf.split(b'\r\n') - if lines and len(lines) > 4: + if lines and len(lines) > 1: hex_items = lines[0].split(b'%') if hex_items and len(hex_items) > 1: for index in range(1, len(hex_items)): @@ -156,7 +156,7 @@ def get_data_from_http_header(self, buf): def get_host_from_http_header(self, buf): ret_buf = b'' lines = buf.split(b'\r\n') - if lines and len(lines) > 4: + if lines and len(lines) > 1: for line in lines: if match_begin(line, b"Host: "): return line[6:] diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 46ecc9ba..a56fae6c 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -62,7 +62,7 @@ def __init__(self, method): self.send_buffer = b'' self.recv_buffer = b'' self.client_id = b'' - self.max_time_dif = 0 # time dif (second) setting + self.max_time_dif = 60 * 60 * 24 # time dif (second) setting self.tls_version = b'\x03\x03' def init_data(self): @@ -215,14 +215,18 @@ def server_decode(self, buf): return self.server_decode(b'') #raise Exception("handshake data = %s" % (binascii.hexlify(buf))) - self.handshake_status = 2 + self.recv_buffer += buf + buf = self.recv_buffer ogn_buf = buf + if len(buf) < 3: + return (b'', False, False) if not match_begin(buf, b'\x16\x03\x01'): return self.decode_error_return(ogn_buf) buf = buf[3:] - if struct.unpack('>H', buf[:2])[0] != len(buf) - 2: - logging.info("tls_auth wrong tls head size") - return self.decode_error_return(ogn_buf) + if struct.unpack('>H', buf[:2])[0] > len(buf) - 2: + return (b'', False, False) + + self.handshake_status = 2 buf = buf[2:] if not match_begin(buf, b'\x01\x00'): #client hello logging.info("tls_auth not client hello message") diff --git a/shadowsocks/server.py b/shadowsocks/server.py index aa67303a..d3ce95cb 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -103,7 +103,7 @@ def main(): a_config = config.copy() ipv6_ok = False logging.info("server start with protocol[%s] password [%s] method [%s] obfs [%s] obfs_param [%s]" % - (protocol, password, a_config['method'], obfs, obfs_param)) + (protocol, password, method, obfs, obfs_param)) if 'server_ipv6' in a_config: try: if len(a_config['server_ipv6']) > 2 and a_config['server_ipv6'][0] == "[" and a_config['server_ipv6'][-1] == "]": diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 5973513c..755263db 100644 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -171,7 +171,7 @@ def get_config(is_local): logging.info('loading config from %s' % config_path) with open(config_path, 'rb') as f: try: - config = parse_json_in_str(f.read().decode('utf8')) + config = parse_json_in_str(remove_comment(f.read().decode('utf8'))) except ValueError as e: logging.error('found an error in config.json: %s', e.message) @@ -410,6 +410,47 @@ def _decode_dict(data): rv[key] = value return rv +class JSFormat: + def __init__(self): + self.state = 0 + + def push(self, ch): + ch = ord(ch) + if self.state == 0: + if ch == ord('"'): + self.state = 1 + return to_str(chr(ch)) + elif ch == ord('/'): + self.state = 3 + else: + return to_str(chr(ch)) + elif self.state == 1: + if ch == ord('"'): + self.state = 0 + return to_str(chr(ch)) + elif ch == ord('\\'): + self.state = 2 + return to_str(chr(ch)) + elif self.state == 2: + self.state = 1 + if ch == ord('"'): + return to_str(chr(ch)) + return "\\" + to_str(chr(ch)) + elif self.state == 3: + if ch == ord('/'): + self.state = 4 + else: + return "/" + to_str(chr(ch)) + elif self.state == 4: + if ch == ord('\n'): + self.state = 0 + return "\n" + return "" + +def remove_comment(json): + fmt = JSFormat() + return "".join([fmt.push(c) for c in json]) + def parse_json_in_str(data): # parse json and convert everything from unicode to str From b513210778f74ac8a91c8f5419ac1326f4f94aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 23 Dec 2016 21:14:23 +0800 Subject: [PATCH 315/487] fix tls --- shadowsocks/obfsplugin/obfs_tls.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index a56fae6c..7646f6e8 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -197,6 +197,8 @@ def server_decode(self, buf): return (ret, True, False) if self.handshake_status == 3: + self.recv_buffer += buf + buf = self.recv_buffer verify = buf verify_len = 43 - 10 if len(buf) < 43: @@ -226,6 +228,7 @@ def server_decode(self, buf): if struct.unpack('>H', buf[:2])[0] > len(buf) - 2: return (b'', False, False) + self.recv_buffer = self.recv_buffer[struct.unpack('>H', buf[:2])[0] + 5:] self.handshake_status = 2 buf = buf[2:] if not match_begin(buf, b'\x01\x00'): #client hello From 403ce5da582fc67f4f24a2626145a9e26074aab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 23 Dec 2016 21:40:38 +0800 Subject: [PATCH 316/487] bump --- CHANGES | 4 ++++ shadowsocks/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index e865e964..4aee5a94 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +3.0.0 2016-12-23 +- http_simple fix bugs +- tls1.2_ticket_auth fix bug & defaule time diff set to 86400s + 2.9.7 2016-11-22 - manage client with LRUCache - catch bind error diff --git a/shadowsocks/version.py b/shadowsocks/version.py index d0befd04..218f2bb9 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '2.9.7 2016-11-22' + return '3.0.0 2016-12-23' From c7815a0ee83cd48365caaa7bd15b1dc33696a43f Mon Sep 17 00:00:00 2001 From: glzjin Date: Sat, 24 Dec 2016 07:25:53 -0600 Subject: [PATCH 317/487] Support redirect rule (#145) Syntax: match_hostname_regex:port(* means match all port)#redirect_dist_host:redirect_port and in the config.json(user-config.json) redirect param should be a list, for example ``` "redirect": ["*:8080#zhaoj.in:80","zhaojin97.cn:80#127.0.0.1:80","*#pku.edu.cn:80"] ``` This example means abnormal connection which connect to port 8080 will be redirected to zhaoj.in:80,and if there a http request with hostname zhaojin97.cn(you can set hosts file to test it) will be redirected to 127.0.0.1:80,and the rest of abnormal connection will be redirect to pku.edu.cn:80 --- shadowsocks/common.py | 8 ++++ shadowsocks/tcprelay.py | 96 ++++++++++++++++++++++++++++------------- 2 files changed, 73 insertions(+), 31 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index e1f574c8..fdb22437 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -22,6 +22,7 @@ import struct import logging import binascii +import re def compat_ord(s): if type(s) == int: @@ -118,6 +119,13 @@ def is_ip(address): return False +def match_regex(regex, text): + regex = re.compile(regex) + for item in regex.findall(text): + return True + return False + + def patch_socket(): if not hasattr(socket, 'inet_pton'): socket.inet_pton = inet_pton diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 66783016..d8c5ca11 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -151,7 +151,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.tcp_mss = 1460 self._protocol.set_server_info(server_info) - self._redir_list = config.get('redirect', ["0.0.0.0:0"]) + self._redir_list = config.get('redirect', ["*#0.0.0.0:0"]) self._bind = config.get('out_bind', '') self._bindv6 = config.get('out_bindv6', '') self._ignore_bind_list = config.get('ignore_bind', []) @@ -347,43 +347,77 @@ def _write_to_sock(self, data, sock): return True def _get_redirect_host(self, client_address, ogn_data): - host_list = self._redir_list or ["0.0.0.0:0"] - hash_code = binascii.crc32(ogn_data) - addrs = socket.getaddrinfo(client_address[0], client_address[1], 0, socket.SOCK_STREAM, socket.SOL_TCP) - af, socktype, proto, canonname, sa = addrs[0] - address_bytes = common.inet_pton(af, sa[0]) - if af == socket.AF_INET6: - addr = struct.unpack('>Q', address_bytes[8:])[0] - elif af == socket.AF_INET: - addr = struct.unpack('>I', address_bytes)[0] - else: - addr = 0 + host_list = self._redir_list or ["*#0.0.0.0:0"] - host_port = [] - match_port = False if type(host_list) != list: host_list = [host_list] - for host in host_list: - items = common.to_str(host).rsplit(':', 1) - if len(items) > 1: - try: - port = int(items[1]) - if port == self._server._listen_port: - match_port = True - host_port.append((items[0], port)) - except: - pass + + items_sum = common.to_str(host_list[0]).rsplit('#', 1) + if len(items_sum) < 2: + hash_code = binascii.crc32(ogn_data) + addrs = socket.getaddrinfo(client_address[0], client_address[1], 0, socket.SOCK_STREAM, socket.SOL_TCP) + af, socktype, proto, canonname, sa = addrs[0] + address_bytes = common.inet_pton(af, sa[0]) + if af == socket.AF_INET6: + addr = struct.unpack('>Q', address_bytes[8:])[0] + elif af == socket.AF_INET: + addr = struct.unpack('>I', address_bytes)[0] else: - host_port.append((host, 80)) + addr = 0 + + host_port = [] + match_port = False + for host in host_list: + items = common.to_str(host).rsplit(':', 1) + if len(items) > 1: + try: + port = int(items[1]) + if port == self._server._listen_port: + match_port = True + host_port.append((items[0], port)) + except: + pass + else: + host_port.append((host, 80)) + + if match_port: + last_host_port = host_port + host_port = [] + for host in last_host_port: + if host[1] == self._server._listen_port: + host_port.append(host) - if match_port: - last_host_port = host_port + return host_port[((hash_code & 0xffffffff) + addr) % len(host_port)] + + else: host_port = [] - for host in last_host_port: - if host[1] == self._server._listen_port: - host_port.append(host) + for host in host_list: + items_sum = common.to_str(host).rsplit('#', 1) + items_match = common.to_str(items_sum[0]).rsplit(':', 1) + items = common.to_str(items_sum[1]).rsplit(':', 1) + if len(items_match) > 1: + if self._server._listen_port != int(items_match[1]): + continue + match_port = 0 + if len(items_match) > 1: + if items_match[1] != "*": + try: + match_port = int(items_match[1]) + except: + pass + if items_match[0] != "*" and common.match_regex(items_match[0], ogn_data) == False and \ + not (match_port == self._server._listen_port or match_port == 0): + continue + if len(items) > 1: + try: + port = int(items[1]) + return (items[0], port) + except: + pass + else: + return (items[0], 80) - return host_port[((hash_code & 0xffffffff) + addr) % len(host_port)] + return ("0.0.0.0", 0) def _handel_protocol_error(self, client_address, ogn_data): logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d via port %d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1], self._server._listen_port)) From 5bd61e1bfb7f819cc9e804211f713705c29c350c Mon Sep 17 00:00:00 2001 From: glzjin Date: Tue, 27 Dec 2016 02:20:38 -0600 Subject: [PATCH 318/487] refine redirect (#147) refine redirect, forbidden list will not work when redirect rule work --- shadowsocks/tcprelay.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index d8c5ca11..d77223b4 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -152,6 +152,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._protocol.set_server_info(server_info) self._redir_list = config.get('redirect', ["*#0.0.0.0:0"]) + self._is_redirect = False self._bind = config.get('out_bind', '') self._bindv6 = config.get('out_bindv6', '') self._ignore_bind_list = config.get('ignore_bind', []) @@ -427,6 +428,7 @@ def _handel_protocol_error(self, client_address, ogn_data): if port == 0: raise Exception('can not parse header') data = b"\x03" + common.to_bytes(common.chr(len(host))) + common.to_bytes(host) + struct.pack('>H', port) + self._is_redirect = True logging.warn("TCP data redir %s:%d %s" % (host, port, binascii.hexlify(data))) return data + ogn_data @@ -596,7 +598,7 @@ def _create_remote_socket(self, ip, port): if len(addrs) == 0: raise Exception("getaddrinfo failed for %s:%d" % (ip, port)) af, socktype, proto, canonname, sa = addrs[0] - if not self._remote_udp: + if not self._remote_udp and not self._is_redirect: if self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: if self._remote_address: From 349821221d6e5f262d72d8c6fb55496a76b6cb7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 29 Dec 2016 14:15:18 +0800 Subject: [PATCH 319/487] force pack some data in url --- shadowsocks/obfsplugin/http_simple.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 596b6497..6549f779 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -198,6 +198,8 @@ def server_decode(self, buf): hosts = self.server_info.obfs_param.split(',') if host not in hosts: return self.not_match_return(buf) + if len(ret_buf) < 4: + return self.not_match_return(buf) if len(datas) > 1: ret_buf += datas[1] if len(ret_buf) >= 7: From 9dfcfe0f7e8355769f1c2feb24af6c33e98a60e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 30 Dec 2016 00:18:16 +0800 Subject: [PATCH 320/487] remove all compatible --- shadowsocks/obfsplugin/auth.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 19e87556..3d23a52d 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -59,18 +59,12 @@ def create_auth_aes128_sha1(method): obfs_map = { 'auth_sha1': (create_auth_sha1,), - 'auth_sha1_compatible': (create_auth_sha1,), 'auth_sha1_v2': (create_auth_sha1_v2,), - 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), 'auth_sha1_v3': (create_auth_sha1_v3,), - 'auth_sha1_v3_compatible': (create_auth_sha1_v3,), 'auth_sha1_v4': (create_auth_sha1_v4,), - 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), 'auth_aes128': (create_auth_aes128,), 'auth_aes128_md5': (create_auth_aes128_md5,), - 'auth_aes128_md5_compatible': (create_auth_aes128_md5,), 'auth_aes128_sha1': (create_auth_aes128_sha1,), - 'auth_aes128_sha1_compatible': (create_auth_aes128_sha1,), } def match_begin(str1, str2): From 19d0034ba905de8d996535a6e7b108525c9edd31 Mon Sep 17 00:00:00 2001 From: orvice Date: Fri, 30 Dec 2016 00:52:06 +0800 Subject: [PATCH 321/487] update README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b6b199f9..769b58e9 100644 --- a/README.md +++ b/README.md @@ -13,16 +13,16 @@ Server Debian / Ubuntu: apt-get install git - git clone https://github.com/breakwa11/shadowsocks.git + git clone https://github.com/shadowsocksr/shadowsocksr.git CentOS: yum install git - git clone https://github.com/breakwa11/shadowsocks.git + git clone https://github.com/shadowsocksr/shadowsocksr.git Windows: - git clone https://github.com/breakwa11/shadowsocks.git + git clone https://github.com/shadowsocksr/shadowsocksr.git ### Usage for single user on linux platform @@ -94,12 +94,12 @@ Bugs and Issues [Android]: https://github.com/shadowsocks/shadowsocks-android -[Build Status]: https://travis-ci.org/falseen/shadowsocks.svg?branch=manyuser-travis +[Build Status]: https://travis-ci.org/shadowsocksr/shadowsocksr.svg?branch=manyuser [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help -[Issue Tracker]: https://github.com/breakwa11/shadowsocks/issues?state=open +[Issue Tracker]: https://github.com/shadowsocksr/shadowsocksr/issues?state=open [OpenWRT]: https://github.com/shadowsocks/openwrt-shadowsocks [OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help -[Travis CI]: https://travis-ci.org/falseen/shadowsocks +[Travis CI]: https://travis-ci.org/shadowsocksr/shadowsocksr [Windows]: https://github.com/breakwa11/shadowsocks-csharp [Wiki]: https://github.com/breakwa11/shadowsocks-rss/wiki \ No newline at end of file From 486c007928adfef3dbeda6b0c4435cad680d333d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 30 Dec 2016 11:22:24 +0800 Subject: [PATCH 322/487] change default protocol settings, refine auth_aes128_* for redir method --- config.json | 2 +- mujson_mgr.py | 9 ++------- shadowsocks/obfsplugin/auth.py | 13 +------------ 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/config.json b/config.json index 572bf580..d9eb7a1b 100644 --- a/config.json +++ b/config.json @@ -8,7 +8,7 @@ "timeout": 120, "udp_timeout": 60, "method": "aes-256-cfb", - "protocol": "auth_sha1_v4_compatible", + "protocol": "auth_sha1_v4", "protocol_param": "", "obfs": "tls1.2_ticket_auth_compatible", "obfs_param": "", diff --git a/mujson_mgr.py b/mujson_mgr.py index 179723e6..5b1424ca 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -92,7 +92,7 @@ def rand_pass(self): def add(self, user): up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-cfb", - 'protocol': "auth_sha1_v4_compatible", + 'protocol': "auth_sha1_v4", 'obfs': "tls1.2_ticket_auth_compatible", 'transfer_enable': 1125899906842624} up['passwd'] = self.rand_pass() @@ -187,7 +187,7 @@ def print_server_help(): -p PORT server port -k PASSWORD password -m METHOD encryption method, default: aes-128-cfb - -O PROTOCOL protocol plugin, default: auth_sha1_v2_compatible + -O PROTOCOL protocol plugin, default: auth_sha1_v4 -o OBFS obfs plugin, default: tls1.2_ticket_auth_compatible -G PROTOCOL_PARAM protocol plugin param -g OBFS_PARAM obfs plugin param @@ -211,16 +211,11 @@ def main(): fast_set_protocol = {'0': 'origin', '+1': 'verify_sha1_compatible', '1': 'verify_sha1', - '+2': 'auth_sha1_compatible', '2': 'auth_sha1', - '+3': 'auth_sha1_v2_compatible', '3': 'auth_sha1_v2', - '+4': 'auth_sha1_v4_compatible', '4': 'auth_sha1_v4', 'am': 'auth_aes128_md5', 'as': 'auth_aes128_sha1', - '+am': 'auth_aes128_md5_compatible', - '+as': 'auth_aes128_sha1_compatible' } fast_set_method = {'a0': 'aes-128-cfb', 'a1': 'aes-192-cfb', diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 3d23a52d..44950403 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -81,7 +81,6 @@ class auth_base(plain.plain): def __init__(self, method): super(auth_base, self).__init__(method) self.method = method - self.no_compatible_method = '' def init_data(self): return '' @@ -103,9 +102,7 @@ def server_decode(self, buf): def not_match_return(self, buf): self.raw_trans = True - if self.method == self.no_compatible_method: - return (b'E'*2048, False) - return (buf, False) + return (b'E'*2048, False) class client_queue(object): def __init__(self, begin_id): @@ -214,7 +211,6 @@ def __init__(self, method): self.client_id = 0 self.connection_id = 0 self.max_time_dif = 60 * 60 # time dif (second) setting - self.no_compatible_method = 'auth_sha1' def init_data(self): return obfs_auth_data() @@ -436,7 +432,6 @@ def __init__(self, method): self.client_id = 0 self.connection_id = 0 self.salt = b"auth_sha1_v2" - self.no_compatible_method = 'auth_sha1_v2' def init_data(self): return obfs_auth_v2_data() @@ -638,7 +633,6 @@ def __init__(self, method): self.connection_id = 0 self.max_time_dif = 60 * 60 * 24 # time dif (second) setting self.salt = b"auth_sha1_v3" - self.no_compatible_method = 'auth_sha1_v3' def init_data(self): return obfs_auth_v2_data() @@ -847,7 +841,6 @@ def __init__(self, method): self.connection_id = 0 self.max_time_dif = 60 * 60 * 24 # time dif (second) setting self.salt = b"auth_sha1_v4" - self.no_compatible_method = 'auth_sha1_v4' def init_data(self): return obfs_auth_v2_data() @@ -1072,7 +1065,6 @@ def __init__(self, method): self.connection_id = 0 self.max_time_dif = 60 * 60 * 24 # time dif (second) setting self.salt = b"auth_aes128" - self.no_compatible_method = 'auth_aes128' self.extra_wait_size = struct.unpack('>H', os.urandom(2))[0] % 1024 self.pack_id = 0 self.recv_id = 0 @@ -1479,9 +1471,6 @@ def server_post_decrypt(self, buf): mac_key = self.server_info.recv_iv + self.server_info.key sha1data = hmac.new(mac_key, self.recv_buf[:1], self.hashfunc).digest()[:6] if sha1data != self.recv_buf[1:7]: - if self.method == self.no_compatible_method: - if len(self.recv_buf) < 31 + self.extra_wait_size: - return (b'', False) return self.not_match_return(self.recv_buf) if len(self.recv_buf) < 31: From e63d1eafad349682f27012fee810cccf45d03ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 31 Dec 2016 21:50:42 +0800 Subject: [PATCH 323/487] auth_sha1 & auth_sha1_v2 keep compatible, remove auth_sha1_v3 --- shadowsocks/obfsplugin/auth.py | 220 ++------------------------------- 1 file changed, 11 insertions(+), 209 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 44950403..54ca9246 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -59,7 +59,9 @@ def create_auth_aes128_sha1(method): obfs_map = { 'auth_sha1': (create_auth_sha1,), + 'auth_sha1_compatible': (create_auth_sha1,), 'auth_sha1_v2': (create_auth_sha1_v2,), + 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), 'auth_sha1_v3': (create_auth_sha1_v3,), 'auth_sha1_v4': (create_auth_sha1_v4,), 'auth_aes128': (create_auth_aes128,), @@ -81,6 +83,7 @@ class auth_base(plain.plain): def __init__(self, method): super(auth_base, self).__init__(method) self.method = method + self.no_compatible_method = '' def init_data(self): return '' @@ -102,7 +105,9 @@ def server_decode(self, buf): def not_match_return(self, buf): self.raw_trans = True - return (b'E'*2048, False) + if self.method == self.no_compatible_method: + return (b'E'*2048, False) + return (buf, False) class client_queue(object): def __init__(self, begin_id): @@ -211,6 +216,7 @@ def __init__(self, method): self.client_id = 0 self.connection_id = 0 self.max_time_dif = 60 * 60 # time dif (second) setting + self.no_compatible_method = 'auth_sha1' def init_data(self): return obfs_auth_data() @@ -432,6 +438,7 @@ def __init__(self, method): self.client_id = 0 self.connection_id = 0 self.salt = b"auth_sha1_v2" + self.no_compatible_method = 'auth_sha1_v2' def init_data(self): return obfs_auth_v2_data() @@ -620,214 +627,6 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, sendback) -class auth_sha1_v3(auth_base): - def __init__(self, method): - super(auth_sha1_v3, self).__init__(method) - self.recv_buf = b'' - self.unit_len = 8100 - self.decrypt_packet_num = 0 - self.raw_trans = False - self.has_sent_header = False - self.has_recv_header = False - self.client_id = 0 - self.connection_id = 0 - self.max_time_dif = 60 * 60 * 24 # time dif (second) setting - self.salt = b"auth_sha1_v3" - - def init_data(self): - return obfs_auth_v2_data() - - def set_server_info(self, server_info): - self.server_info = server_info - try: - max_client = int(server_info.protocol_param) - except: - max_client = 64 - self.server_info.data.set_max_client(max_client) - - def rnd_data(self, buf_size): - if buf_size > 1200: - return b'\x01' - - if buf_size > 400: - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 256) - else: - rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) - - if len(rnd_data) < 128: - return common.chr(len(rnd_data) + 1) + rnd_data - else: - return common.chr(255) + struct.pack('>H', len(rnd_data) + 3) + rnd_data - - def pack_data(self, buf): - data = self.rnd_data(len(buf)) + buf - data = struct.pack('>H', len(data) + 6) + data - adler32 = zlib.adler32(data) & 0xFFFFFFFF - data += struct.pack('H', data_len)) & 0xFFFFFFFF - data = struct.pack('H', data_len) + data - data += hmac.new(self.server_info.iv + self.server_info.key, data, hashlib.sha1).digest()[:10] - return data - - def auth_data(self): - utc_time = int(time.time()) & 0xFFFFFFFF - if self.server_info.data.connection_id > 0xFF000000: - self.server_info.data.local_client_id = b'' - if not self.server_info.data.local_client_id: - self.server_info.data.local_client_id = os.urandom(4) - logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) - self.server_info.data.connection_id = struct.unpack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def client_post_decrypt(self, buf): - if self.raw_trans: - return buf - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data error') - if length > len(self.recv_buf): - break - - if struct.pack('H', self.recv_buf[3:5])[0] + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - - if out_buf: - self.decrypt_packet_num += 1 - return out_buf - - def server_pre_encrypt(self, buf): - if self.raw_trans: - return buf - ret = b'' - while len(buf) > self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def server_post_decrypt(self, buf): - if self.raw_trans: - return (buf, False) - self.recv_buf += buf - out_buf = b'' - if not self.has_recv_header: - if len(self.recv_buf) < 6: - return (b'', False) - crc = struct.pack('H', self.recv_buf[:2])[0] - if length > len(self.recv_buf): - return (b'', False) - sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] - if sha1data != self.recv_buf[length - 10:length]: - logging.error('auth_sha1_v3 data uncorrect auth HMAC-SHA1') - return self.not_match_return(self.recv_buf) - pos = common.ord(self.recv_buf[6]) - if pos < 255: - pos += 6 - else: - pos = struct.unpack('>H', self.recv_buf[7:9])[0] + 6 - out_buf = self.recv_buf[pos:length - 10] - if len(out_buf) < 12: - logging.info('auth_sha1_v3: too short, data %s' % (binascii.hexlify(self.recv_buf),)) - return self.not_match_return(self.recv_buf) - utc_time = struct.unpack(' self.max_time_dif: - logging.info('auth_sha1_v3: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) - return self.not_match_return(self.recv_buf) - elif self.server_info.data.insert(client_id, connection_id): - self.has_recv_header = True - out_buf = out_buf[12:] - self.client_id = client_id - self.connection_id = connection_id - else: - logging.info('auth_sha1_v3: auth fail, data %s' % (binascii.hexlify(out_buf),)) - return self.not_match_return(self.recv_buf) - self.recv_buf = self.recv_buf[length:] - self.has_recv_header = True - - sendback = False - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - logging.info('auth_sha1_v3: over size') - return (b'E'*2048, False) - else: - raise Exception('server_post_decrype data error') - if length > len(self.recv_buf): - break - - if struct.pack('H', self.recv_buf[3:5])[0] + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - if pos == length - 4: - sendback = True - - if out_buf: - self.server_info.data.update(self.client_id, self.connection_id) - self.decrypt_packet_num += 1 - return (out_buf, sendback) - class auth_sha1_v4(auth_base): def __init__(self, method): super(auth_sha1_v4, self).__init__(method) @@ -841,6 +640,7 @@ def __init__(self, method): self.connection_id = 0 self.max_time_dif = 60 * 60 * 24 # time dif (second) setting self.salt = b"auth_sha1_v4" + self.no_compatible_method = 'auth_sha1_v4' def init_data(self): return obfs_auth_v2_data() @@ -1065,6 +865,7 @@ def __init__(self, method): self.connection_id = 0 self.max_time_dif = 60 * 60 * 24 # time dif (second) setting self.salt = b"auth_aes128" + self.no_compatible_method = 'auth_aes128' self.extra_wait_size = struct.unpack('>H', os.urandom(2))[0] % 1024 self.pack_id = 0 self.recv_id = 0 @@ -1583,3 +1384,4 @@ def server_udp_post_decrypt(self, buf): if hmac.new(user_key, buf[:-4], self.hashfunc).digest()[:4] != buf[-4:]: return b'' return buf[:-8] + From 33862fa8d31708c15dbb059cd3fde2718c322ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 31 Dec 2016 23:16:29 +0800 Subject: [PATCH 324/487] remove auth_sha1_v3 --- shadowsocks/obfsplugin/auth.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 54ca9246..5d4e748c 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -42,9 +42,6 @@ def create_auth_sha1(method): def create_auth_sha1_v2(method): return auth_sha1_v2(method) -def create_auth_sha1_v3(method): - return auth_sha1_v3(method) - def create_auth_sha1_v4(method): return auth_sha1_v4(method) @@ -62,7 +59,6 @@ def create_auth_aes128_sha1(method): 'auth_sha1_compatible': (create_auth_sha1,), 'auth_sha1_v2': (create_auth_sha1_v2,), 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), - 'auth_sha1_v3': (create_auth_sha1_v3,), 'auth_sha1_v4': (create_auth_sha1_v4,), 'auth_aes128': (create_auth_aes128,), 'auth_aes128_md5': (create_auth_aes128_md5,), From 5ae0c67965332522635d994914321bdd59648139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 2 Jan 2017 00:50:51 +0800 Subject: [PATCH 325/487] add auth_sha1_v4_compatible --- shadowsocks/obfsplugin/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 5d4e748c..081fe9d2 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -60,6 +60,7 @@ def create_auth_aes128_sha1(method): 'auth_sha1_v2': (create_auth_sha1_v2,), 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), 'auth_sha1_v4': (create_auth_sha1_v4,), + 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), 'auth_aes128': (create_auth_aes128,), 'auth_aes128_md5': (create_auth_aes128_md5,), 'auth_aes128_sha1': (create_auth_aes128_sha1,), From 77155a5ee4402d8fcaa59059de68d4b3f38fe462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 3 Jan 2017 11:44:07 +0800 Subject: [PATCH 326/487] bump --- CHANGES | 3 +++ shadowsocks/version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 4aee5a94..ee2741e7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +3.0.1 2017-01-03 +- remove auth_aes128_*_compatible + 3.0.0 2016-12-23 - http_simple fix bugs - tls1.2_ticket_auth fix bug & defaule time diff set to 86400s diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 218f2bb9..6bab5d7f 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.0.0 2016-12-23' + return '3.0.1 2017-01-03' From 7c6d059f0c0a1bd67246f82e6f5b3d09de6b4d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 4 Jan 2017 01:25:23 +0800 Subject: [PATCH 327/487] change default protocol to auth_aes128_md5 --- config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json b/config.json index d9eb7a1b..199b111c 100644 --- a/config.json +++ b/config.json @@ -8,7 +8,7 @@ "timeout": 120, "udp_timeout": 60, "method": "aes-256-cfb", - "protocol": "auth_sha1_v4", + "protocol": "auth_aes128_md5", "protocol_param": "", "obfs": "tls1.2_ticket_auth_compatible", "obfs_param": "", From 22ce739a45bd6a77c88d3384edb654bf979a90e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 4 Jan 2017 19:40:47 +0800 Subject: [PATCH 328/487] update readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 769b58e9..14ad3e25 100644 --- a/README.md +++ b/README.md @@ -26,18 +26,18 @@ Windows: ### Usage for single user on linux platform -If you clone it into "~/shadowsocks" -move to "~/shadowsocks", then run: +If you clone it into "~/shadowsocksr" +move to "~/shadowsocksr", then run: bash initcfg.sh -move to "~/shadowsocks/shadowsocks", then run: +move to "~/shadowsocksr/shadowsocks", then run: python server.py -p 443 -k password -m aes-128-cfb -O auth_aes128_md5_compatible -o tls1.2_ticket_auth_compatible Check all the options via `-h`. -You can also use a configuration file instead (recommand), move to "~/shadowsocks" and edit the file "user-config.json", then move to "~/shadowsocks/shadowsocks" again, just run: +You can also use a configuration file instead (recommand), move to "~/shadowsocksr" and edit the file "user-config.json", then move to "~/shadowsocksr/shadowsocks" again, just run: python server.py @@ -93,7 +93,7 @@ Bugs and Issues -[Android]: https://github.com/shadowsocks/shadowsocks-android +[Android]: https://github.com/shadowsocksr/shadowsocksr-android [Build Status]: https://travis-ci.org/shadowsocksr/shadowsocksr.svg?branch=manyuser [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help @@ -101,5 +101,5 @@ Bugs and Issues [OpenWRT]: https://github.com/shadowsocks/openwrt-shadowsocks [OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help [Travis CI]: https://travis-ci.org/shadowsocksr/shadowsocksr -[Windows]: https://github.com/breakwa11/shadowsocks-csharp +[Windows]: https://github.com/shadowsocksr/shadowsocksr-csharp [Wiki]: https://github.com/breakwa11/shadowsocks-rss/wiki \ No newline at end of file From 959aad3f412808f6fd2aa522a790ce29786cf60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 4 Jan 2017 22:46:39 +0800 Subject: [PATCH 329/487] multiuser in single port protocol --- shadowsocks/obfsplugin/auth.py | 31 ++++++++---- shadowsocks/obfsplugin/plain.py | 5 +- shadowsocks/obfsplugin/verify.py | 8 +-- shadowsocks/tcprelay.py | 48 +++++++++++++++++- shadowsocks/udprelay.py | 84 ++++++++++++++++++++++++++------ 5 files changed, 144 insertions(+), 32 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 081fe9d2..02446801 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1103,8 +1103,8 @@ def server_udp_post_decrypt(self, buf): length = len(buf) data = buf[:-4] if struct.pack(' Date: Fri, 6 Jan 2017 21:02:52 +0800 Subject: [PATCH 330/487] update MU auth_aes128_* protocol --- shadowsocks/obfsplugin/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 02446801..8edd6a01 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1282,7 +1282,7 @@ def server_post_decrypt(self, buf): uid = self.recv_buf[7:11] if uid in self.server_info.users: - self.user_key = self.server_info.users[uid] + self.user_key = self.hashfunc(self.server_info.users[uid]).digest() self.server_info.update_user_func(uid) else: if not self.server_info.users: @@ -1384,7 +1384,7 @@ def server_udp_pre_encrypt(self, buf): def server_udp_post_decrypt(self, buf): uid = buf[-8:-4] if uid in self.server_info.users: - self.user_key = self.server_info.users[uid] + self.user_key = self.hashfunc(self.server_info.users[uid]).digest() else: uid = None if not self.server_info.users: From b367ad0560bb1a91e836f6d18f0014b6232538d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 7 Jan 2017 01:23:42 +0800 Subject: [PATCH 331/487] using '#' to split multi-user setting --- shadowsocks/obfsplugin/auth.py | 2 +- shadowsocks/tcprelay.py | 18 ++++++++++-------- shadowsocks/udprelay.py | 18 ++++++++++-------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 8edd6a01..4150aaa7 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1131,7 +1131,7 @@ def init_data(self): def set_server_info(self, server_info): self.server_info = server_info try: - max_client = int(server_info.protocol_param) + max_client = int(server_info.protocol_param.split('#')[0]) except: max_client = 64 self.server_info.data.set_max_client(max_client) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 763122d0..8db91f9a 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1020,14 +1020,16 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self._listen_port = listen_port if config['protocol'] in ["auth_aes128_md5", "auth_aes128_sha1"]: - user_list = config['protocol_param'].split(',') - if user_list: - for user in user_list: - items = user.split(':') - if len(items) == 2: - uid = struct.pack(' Date: Sat, 7 Jan 2017 18:50:12 +1100 Subject: [PATCH 332/487] Use space instead of tab for indent see PEP8 --- mujson_mgr.py | 523 +++++++++++++++++++++++++------------------------- 1 file changed, 266 insertions(+), 257 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 5b1424ca..a40c8fee 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -10,167 +10,175 @@ import json import base64 + class MuJsonLoader(object): - def __init__(self): - self.json = None + def __init__(self): + self.json = None + + def load(self, path): + with open(path, 'rb+') as f: + self.json = json.loads(f.read().decode('utf8')) - def load(self, path): - with open(path, 'rb+') as f: - self.json = json.loads(f.read().decode('utf8')) + def save(self, path): + if self.json: + output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) + with open(path, 'r+') as f: + f.write(output) + f.truncate() - def save(self, path): - if self.json: - output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) - with open(path, 'r+') as f: - f.write(output) - f.truncate() class MuMgr(object): - def __init__(self): - self.config_path = get_config().MUDB_FILE - try: - self.server_addr = get_config().SERVER_PUB_ADDR - except: - self.server_addr = '127.0.0.1' - self.data = MuJsonLoader() + def __init__(self): + self.config_path = get_config().MUDB_FILE + try: + self.server_addr = get_config().SERVER_PUB_ADDR + except: + self.server_addr = '127.0.0.1' + self.data = MuJsonLoader() + + if self.server_addr == '127.0.0.1': + self.server_addr = self.getipaddr() - if self.server_addr == '127.0.0.1': - self.server_addr = self.getipaddr() + def getipaddr(self, ifname='eth0'): + import socket + import struct + import fcntl + ret = '127.0.0.1' + try: + ret = socket.gethostbyname(socket.getfqdn(socket.gethostname())) + except: + pass + if ret == '127.0.0.1': + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ret = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24]) + except: + pass + return ret - def getipaddr(self, ifname = 'eth0'): - import socket - import struct - import fcntl - ret = '127.0.0.1' - try: - ret = socket.gethostbyname(socket.getfqdn(socket.gethostname())) - except: - pass - if ret == '127.0.0.1': - try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ret = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24]) - except: - pass - return ret + def ssrlink(self, user, encode): + protocol = user.get('protocol', '') + obfs = user.get('obfs', '') + protocol = protocol.replace("_compatible", "") + obfs = obfs.replace("_compatible", "") + link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, + common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace( + "=", "")) + return "ssr://" + ( + encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) - def ssrlink(self, user, encode): - protocol = user.get('protocol', '') - obfs = user.get('obfs', '') - protocol = protocol.replace("_compatible", "") - obfs = obfs.replace("_compatible", "") - link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace("=", "")) - return "ssr://" + ( encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) + def userinfo(self, user): + ret = "" + for key in sorted(user): + if key in ['enable']: + continue + ret += '\n' + if key in ['transfer_enable', 'u', 'd']: + val = user[key] + if val / 1024 < 4: + ret += " %s : %s" % (key, val) + elif val / 1024 ** 2 < 4: + val /= float(1024) + ret += " %s : %s K Bytes" % (key, val) + elif val / 1024 ** 3 < 4: + val /= float(1024 ** 2) + ret += " %s : %s M Bytes" % (key, val) + else: + val /= float(1024 ** 3) + ret += " %s : %s G Bytes" % (key, val) + else: + ret += " %s : %s" % (key, user[key]) + ret += "\n " + self.ssrlink(user, False) + ret += "\n " + self.ssrlink(user, True) + return ret - def userinfo(self, user): - ret = "" - for key in sorted(user): - if key in ['enable']: - continue - ret += '\n' - if key in ['transfer_enable', 'u', 'd'] : - val = user[key] - if val / 1024 < 4: - ret += " %s : %s" % (key, val) - elif val / 1024**2 < 4: - val /= float(1024) - ret += " %s : %s K Bytes" % (key, val) - elif val / 1024**3 < 4: - val /= float(1024**2) - ret += " %s : %s M Bytes" % (key, val) - else: - val /= float(1024**3) - ret += " %s : %s G Bytes" % (key, val) - else: - ret += " %s : %s" % (key, user[key]) - ret += "\n " + self.ssrlink(user, False) - ret += "\n " + self.ssrlink(user, True) - return ret + def rand_pass(self): + return ''.join( + [random.choice('''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i + in range(8)]) - def rand_pass(self): - return ''.join([random.choice('''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i in range(8)]) + def add(self, user): + up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-cfb", + 'protocol': "auth_sha1_v4", + 'obfs': "tls1.2_ticket_auth_compatible", + 'transfer_enable': 1125899906842624} + up['passwd'] = self.rand_pass() + up.update(user) - def add(self, user): - up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-cfb", - 'protocol': "auth_sha1_v4", - 'obfs': "tls1.2_ticket_auth_compatible", - 'transfer_enable': 1125899906842624} - up['passwd'] = self.rand_pass() - up.update(user) + self.data.load(self.config_path) + for row in self.data.json: + match = False + if 'user' in user and row['user'] == user['user']: + match = True + if 'port' in user and row['port'] == user['port']: + match = True + if match: + print("user [%s] port [%s] already exist" % (row['user'], row['port'])) + return + self.data.json.append(up) + print("### add user info %s" % self.userinfo(up)) + self.data.save(self.config_path) - self.data.load(self.config_path) - for row in self.data.json: - match = False - if 'user' in user and row['user'] == user['user']: - match = True - if 'port' in user and row['port'] == user['port']: - match = True - if match: - print("user [%s] port [%s] already exist" % (row['user'], row['port'])) - return - self.data.json.append(up) - print("### add user info %s" % self.userinfo(up)) - self.data.save(self.config_path) + def edit(self, user): + self.data.load(self.config_path) + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("edit user [%s]" % (row['user'],)) + row.update(user) + print("### new user info %s" % self.userinfo(row)) + break + self.data.save(self.config_path) - def edit(self, user): - self.data.load(self.config_path) - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - print("edit user [%s]" % (row['user'],)) - row.update(user) - print("### new user info %s" % self.userinfo(row)) - break - self.data.save(self.config_path) + def delete(self, user): + self.data.load(self.config_path) + index = 0 + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("delete user [%s]" % row['user']) + del self.data.json[index] + break + index += 1 + self.data.save(self.config_path) - def delete(self, user): - self.data.load(self.config_path) - index = 0 - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - print("delete user [%s]" % row['user']) - del self.data.json[index] - break - index += 1 - self.data.save(self.config_path) + def clear_ud(self, user): + up = {'u': 0, 'd': 0} + self.data.load(self.config_path) + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + row.update(up) + print("clear user [%s]" % row['user']) + self.data.save(self.config_path) - def clear_ud(self, user): - up = {'u': 0, 'd': 0} - self.data.load(self.config_path) - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - row.update(up) - print("clear user [%s]" % row['user']) - self.data.save(self.config_path) + def list_user(self, user): + self.data.load(self.config_path) + if not user: + for row in self.data.json: + print("user [%s] port %s" % (row['user'], row['port'])) + return + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("### user [%s] info %s" % (row['user'], self.userinfo(row))) - def list_user(self, user): - self.data.load(self.config_path) - if not user: - for row in self.data.json: - print("user [%s] port %s" % (row['user'], row['port'])) - return - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - print("### user [%s] info %s" % (row['user'], self.userinfo(row))) def print_server_help(): print('''usage: python mujson_manage.py -a|-d|-e|-c|-l [OPTION]... @@ -198,119 +206,120 @@ def print_server_help(): -h, --help show this help message and exit ''') + def main(): - shortopts = 'adeclu:p:k:O:o:G:g:m:t:f:h' - longopts = ['help'] - action = None - user = {} - fast_set_obfs = {'0': 'plain', - '+1': 'http_simple_compatible', - '1': 'http_simple', - '+2': 'tls1.2_ticket_auth_compatible', - '2': 'tls1.2_ticket_auth'} - fast_set_protocol = {'0': 'origin', - '+1': 'verify_sha1_compatible', - '1': 'verify_sha1', - '2': 'auth_sha1', - '3': 'auth_sha1_v2', - '4': 'auth_sha1_v4', - 'am': 'auth_aes128_md5', - 'as': 'auth_aes128_sha1', - } - fast_set_method = {'a0': 'aes-128-cfb', - 'a1': 'aes-192-cfb', - 'a2': 'aes-256-cfb', - 'r': 'rc4-md5', - 'r6': 'rc4-md5-6', - 'c': 'chacha20', - 'ci': 'chacha20-ietf', - 's': 'salsa20', - 'b': 'bf-cfb', - 'm0': 'camellia-128-cfb', - 'm1': 'camellia-192-cfb', - 'm2': 'camellia-256-cfb', - 'a0t': 'aes-128-ctr', - 'a1t': 'aes-192-ctr', - 'a2t': 'aes-256-ctr'} - try: - optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) - for key, value in optlist: - if key == '-a': - action = 1 - elif key == '-d': - action = 2 - elif key == '-e': - action = 3 - elif key == '-l': - action = 4 - elif key == '-c': - action = 0 - elif key == '-u': - user['user'] = value - elif key == '-p': - user['port'] = int(value) - elif key == '-k': - user['passwd'] = value - elif key == '-o': - if value in fast_set_obfs: - user['obfs'] = fast_set_obfs[value] - else: - user['obfs'] = value - elif key == '-O': - if value in fast_set_protocol: - user['protocol'] = fast_set_protocol[value] - else: - user['protocol'] = value - elif key == '-g': - user['obfs_param'] = value - elif key == '-G': - user['protocol_param'] = value - elif key == '-m': - if value in fast_set_method: - user['method'] = fast_set_method[value] - else: - user['method'] = value - elif key == '-f': - user['forbidden_port'] = value - elif key == '-t': - val = float(value) - try: - val = int(value) - except: - pass - user['transfer_enable'] = val * (1024 ** 3) - elif key in ('-h', '--help'): - print_server_help() - sys.exit(0) - except getopt.GetoptError as e: - print(e) - sys.exit(2) + shortopts = 'adeclu:p:k:O:o:G:g:m:t:f:h' + longopts = ['help'] + action = None + user = {} + fast_set_obfs = {'0': 'plain', + '+1': 'http_simple_compatible', + '1': 'http_simple', + '+2': 'tls1.2_ticket_auth_compatible', + '2': 'tls1.2_ticket_auth'} + fast_set_protocol = {'0': 'origin', + '+1': 'verify_sha1_compatible', + '1': 'verify_sha1', + '2': 'auth_sha1', + '3': 'auth_sha1_v2', + '4': 'auth_sha1_v4', + 'am': 'auth_aes128_md5', + 'as': 'auth_aes128_sha1', + } + fast_set_method = {'a0': 'aes-128-cfb', + 'a1': 'aes-192-cfb', + 'a2': 'aes-256-cfb', + 'r': 'rc4-md5', + 'r6': 'rc4-md5-6', + 'c': 'chacha20', + 'ci': 'chacha20-ietf', + 's': 'salsa20', + 'b': 'bf-cfb', + 'm0': 'camellia-128-cfb', + 'm1': 'camellia-192-cfb', + 'm2': 'camellia-256-cfb', + 'a0t': 'aes-128-ctr', + 'a1t': 'aes-192-ctr', + 'a2t': 'aes-256-ctr'} + try: + optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + for key, value in optlist: + if key == '-a': + action = 1 + elif key == '-d': + action = 2 + elif key == '-e': + action = 3 + elif key == '-l': + action = 4 + elif key == '-c': + action = 0 + elif key == '-u': + user['user'] = value + elif key == '-p': + user['port'] = int(value) + elif key == '-k': + user['passwd'] = value + elif key == '-o': + if value in fast_set_obfs: + user['obfs'] = fast_set_obfs[value] + else: + user['obfs'] = value + elif key == '-O': + if value in fast_set_protocol: + user['protocol'] = fast_set_protocol[value] + else: + user['protocol'] = value + elif key == '-g': + user['obfs_param'] = value + elif key == '-G': + user['protocol_param'] = value + elif key == '-m': + if value in fast_set_method: + user['method'] = fast_set_method[value] + else: + user['method'] = value + elif key == '-f': + user['forbidden_port'] = value + elif key == '-t': + val = float(value) + try: + val = int(value) + except: + pass + user['transfer_enable'] = val * (1024 ** 3) + elif key in ('-h', '--help'): + print_server_help() + sys.exit(0) + except getopt.GetoptError as e: + print(e) + sys.exit(2) - manage = MuMgr() - if action == 0: - manage.clear_ud(user) - elif action == 1: - if 'user' not in user and 'port' in user: - user['user'] = str(user['port']) - if 'user' in user and 'port' in user: - manage.add(user) - else: - print("You have to set the port with -p") - elif action == 2: - if 'user' in user or 'port' in user: - manage.delete(user) - else: - print("You have to set the user name or port with -u/-p") - elif action == 3: - if 'user' in user or 'port' in user: - manage.edit(user) - else: - print("You have to set the user name or port with -u/-p") - elif action == 4: - manage.list_user(user) - elif action is None: - print_server_help() + manage = MuMgr() + if action == 0: + manage.clear_ud(user) + elif action == 1: + if 'user' not in user and 'port' in user: + user['user'] = str(user['port']) + if 'user' in user and 'port' in user: + manage.add(user) + else: + print("You have to set the port with -p") + elif action == 2: + if 'user' in user or 'port' in user: + manage.delete(user) + else: + print("You have to set the user name or port with -u/-p") + elif action == 3: + if 'user' in user or 'port' in user: + manage.edit(user) + else: + print("You have to set the user name or port with -u/-p") + elif action == 4: + manage.list_user(user) + elif action is None: + print_server_help() -if __name__ == '__main__': - main() +if __name__ == '__main__': + main() From 84fd119f26ad16349d55a2f1b69f0f8c8ccd867b Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 19:14:02 +1100 Subject: [PATCH 333/487] Fix mujson_mgr.py crash on windows --- mujson_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index a40c8fee..ca62a842 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -42,7 +42,6 @@ def __init__(self): def getipaddr(self, ifname='eth0'): import socket import struct - import fcntl ret = '127.0.0.1' try: ret = socket.gethostbyname(socket.getfqdn(socket.gethostname())) @@ -50,6 +49,7 @@ def getipaddr(self, ifname='eth0'): pass if ret == '127.0.0.1': try: + import fcntl s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) ret = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24]) except: From 490ae1212832f4c9f86e70e146a782d82edb3e9b Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 19:14:58 +1100 Subject: [PATCH 334/487] Fix mujson_mgr.py crash when MUDB_FILE doesn't exist --- mujson_mgr.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index ca62a842..5dbd44b4 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -16,15 +16,19 @@ def __init__(self): self.json = None def load(self, path): - with open(path, 'rb+') as f: - self.json = json.loads(f.read().decode('utf8')) + l = "[]" + try: + with open(path, 'rb+') as f: + l = f.read().decode('utf8') + except: + pass + self.json = json.loads(l) def save(self, path): if self.json: output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) - with open(path, 'r+') as f: + with open(path, 'w') as f: f.write(output) - f.truncate() class MuMgr(object): From 498f71a21be5e0bb4f454d909220d141a07a689c Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 19:22:36 +1100 Subject: [PATCH 335/487] Save json config as utf8 --- mujson_mgr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 5dbd44b4..62d83169 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -27,8 +27,8 @@ def load(self, path): def save(self, path): if self.json: output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) - with open(path, 'w') as f: - f.write(output) + with open(path, 'wb') as f: + f.write(output.encode('utf8')) class MuMgr(object): From 0f8dd36297e6a0540ab9ce230cc296bfce85db8c Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 19:23:09 +1100 Subject: [PATCH 336/487] Add initcfg.bat for windows --- initcfg.bat | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 initcfg.bat diff --git a/initcfg.bat b/initcfg.bat new file mode 100644 index 00000000..6d0bce69 --- /dev/null +++ b/initcfg.bat @@ -0,0 +1,4 @@ +@echo off +If Not Exist "userapiconfig.py" Copy "apiconfig.py" "userapiconfig.py" +If Not Exist "user-config.json" Copy "config.json" "user-config.json" +If Not Exist "usermysql.json" Copy "mysql.json" "usermysql.json" From 0bf1e46ace5f4b7c0c0cff6de06513256e0e94be Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 19:23:30 +1100 Subject: [PATCH 337/487] Update .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 6c1b61ea..529cc258 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,8 @@ htmlcov .DS_Store .idea + +# Ignore user config +user-config.json +userapiconfig.py +usermysql.json From f05bea4e101c7f1aab0122eb83e2b68089327807 Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 19:34:39 +1100 Subject: [PATCH 338/487] Fix crash on windows --- db_transfer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index dcc158ea..2ef5829c 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -198,8 +198,13 @@ def thread_db(obj): db_instance = obj() ServerPool.get_instance() shell.log_shadowsocks_version() - import resource - logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) + + try: + import resource + logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE)) + except: + pass + try: while True: load_config() From 8777152ec9e75376410117031904f7de5649d104 Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 19:58:32 +1100 Subject: [PATCH 339/487] Run db thread as daemon so that all sub-thread will inherite this and be killed automatically if main thread crash --- server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server.py b/server.py index 4636721e..e6c3dc65 100644 --- a/server.py +++ b/server.py @@ -32,6 +32,7 @@ class MainThread(threading.Thread): def __init__(self, obj): threading.Thread.__init__(self) + self.daemon = True self.obj = obj def run(self): From 6d53546f6ec6ff3a34d0989fdd358f188a7367aa Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 20:03:20 +1100 Subject: [PATCH 340/487] Use super() call --- server.py | 2 +- server_pool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index e6c3dc65..a6541da7 100644 --- a/server.py +++ b/server.py @@ -31,7 +31,7 @@ class MainThread(threading.Thread): def __init__(self, obj): - threading.Thread.__init__(self) + super(MainThread, self).__init__() self.daemon = True self.obj = obj diff --git a/server_pool.py b/server_pool.py index 1fc23673..a526bd05 100644 --- a/server_pool.py +++ b/server_pool.py @@ -32,7 +32,7 @@ class MainThread(threading.Thread): def __init__(self, params): - threading.Thread.__init__(self) + super(MainThread, self).__init__() self.params = params def run(self): From f0b0a31baf28405d3877c4e6a4e8f561d865dece Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 20:27:27 +1100 Subject: [PATCH 341/487] Revert commit fd9214b3343051a0c0c16d3d5d8d30c432dc8590 --- mujson_mgr.py | 528 +++++++++++++++++++++++++------------------------- 1 file changed, 261 insertions(+), 267 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 62d83169..a20a91fd 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -12,180 +12,175 @@ class MuJsonLoader(object): - def __init__(self): - self.json = None + def __init__(self): + self.json = None - def load(self, path): - l = "[]" - try: - with open(path, 'rb+') as f: - l = f.read().decode('utf8') - except: - pass - self.json = json.loads(l) + def load(self, path): + l = "[]" + try: + with open(path, 'rb+') as f: + l = f.read().decode('utf8') + except: + pass + self.json = json.loads(l) - def save(self, path): - if self.json: - output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) - with open(path, 'wb') as f: - f.write(output.encode('utf8')) + def save(self, path): + if self.json: + output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) + with open(path, 'wb') as f: + f.write(output.encode('utf8')) class MuMgr(object): - def __init__(self): - self.config_path = get_config().MUDB_FILE - try: - self.server_addr = get_config().SERVER_PUB_ADDR - except: - self.server_addr = '127.0.0.1' - self.data = MuJsonLoader() + def __init__(self): + self.config_path = get_config().MUDB_FILE + try: + self.server_addr = get_config().SERVER_PUB_ADDR + except: + self.server_addr = '127.0.0.1' + self.data = MuJsonLoader() - if self.server_addr == '127.0.0.1': - self.server_addr = self.getipaddr() + if self.server_addr == '127.0.0.1': + self.server_addr = self.getipaddr() - def getipaddr(self, ifname='eth0'): - import socket - import struct - ret = '127.0.0.1' - try: - ret = socket.gethostbyname(socket.getfqdn(socket.gethostname())) - except: - pass - if ret == '127.0.0.1': - try: - import fcntl - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ret = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24]) - except: - pass - return ret + def getipaddr(self, ifname='eth0'): + import socket + import struct + ret = '127.0.0.1' + try: + ret = socket.gethostbyname(socket.getfqdn(socket.gethostname())) + except: + pass + if ret == '127.0.0.1': + try: + import fcntl + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ret = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24]) + except: + pass + return ret - def ssrlink(self, user, encode): - protocol = user.get('protocol', '') - obfs = user.get('obfs', '') - protocol = protocol.replace("_compatible", "") - obfs = obfs.replace("_compatible", "") - link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, - common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace( - "=", "")) - return "ssr://" + ( - encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) + def ssrlink(self, user, encode): + protocol = user.get('protocol', '') + obfs = user.get('obfs', '') + protocol = protocol.replace("_compatible", "") + obfs = obfs.replace("_compatible", "") + link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace("=", "")) + return "ssr://" + (encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) - def userinfo(self, user): - ret = "" - for key in sorted(user): - if key in ['enable']: - continue - ret += '\n' - if key in ['transfer_enable', 'u', 'd']: - val = user[key] - if val / 1024 < 4: - ret += " %s : %s" % (key, val) - elif val / 1024 ** 2 < 4: - val /= float(1024) - ret += " %s : %s K Bytes" % (key, val) - elif val / 1024 ** 3 < 4: - val /= float(1024 ** 2) - ret += " %s : %s M Bytes" % (key, val) - else: - val /= float(1024 ** 3) - ret += " %s : %s G Bytes" % (key, val) - else: - ret += " %s : %s" % (key, user[key]) - ret += "\n " + self.ssrlink(user, False) - ret += "\n " + self.ssrlink(user, True) - return ret + def userinfo(self, user): + ret = "" + for key in sorted(user): + if key in ['enable']: + continue + ret += '\n' + if key in ['transfer_enable', 'u', 'd']: + val = user[key] + if val / 1024 < 4: + ret += " %s : %s" % (key, val) + elif val / 1024 ** 2 < 4: + val /= float(1024) + ret += " %s : %s K Bytes" % (key, val) + elif val / 1024 ** 3 < 4: + val /= float(1024 ** 2) + ret += " %s : %s M Bytes" % (key, val) + else: + val /= float(1024 ** 3) + ret += " %s : %s G Bytes" % (key, val) + else: + ret += " %s : %s" % (key, user[key]) + ret += "\n " + self.ssrlink(user, False) + ret += "\n " + self.ssrlink(user, True) + return ret - def rand_pass(self): - return ''.join( - [random.choice('''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i - in range(8)]) + def rand_pass(self): + return ''.join([random.choice('''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i in range(8)]) - def add(self, user): - up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-cfb", - 'protocol': "auth_sha1_v4", - 'obfs': "tls1.2_ticket_auth_compatible", - 'transfer_enable': 1125899906842624} - up['passwd'] = self.rand_pass() - up.update(user) + def add(self, user): + up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-cfb", + 'protocol': "auth_sha1_v4", + 'obfs': "tls1.2_ticket_auth_compatible", + 'transfer_enable': 1125899906842624} + up['passwd'] = self.rand_pass() + up.update(user) - self.data.load(self.config_path) - for row in self.data.json: - match = False - if 'user' in user and row['user'] == user['user']: - match = True - if 'port' in user and row['port'] == user['port']: - match = True - if match: - print("user [%s] port [%s] already exist" % (row['user'], row['port'])) - return - self.data.json.append(up) - print("### add user info %s" % self.userinfo(up)) - self.data.save(self.config_path) + self.data.load(self.config_path) + for row in self.data.json: + match = False + if 'user' in user and row['user'] == user['user']: + match = True + if 'port' in user and row['port'] == user['port']: + match = True + if match: + print("user [%s] port [%s] already exist" % (row['user'], row['port'])) + return + self.data.json.append(up) + print("### add user info %s" % self.userinfo(up)) + self.data.save(self.config_path) - def edit(self, user): - self.data.load(self.config_path) - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - print("edit user [%s]" % (row['user'],)) - row.update(user) - print("### new user info %s" % self.userinfo(row)) - break - self.data.save(self.config_path) + def edit(self, user): + self.data.load(self.config_path) + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("edit user [%s]" % (row['user'],)) + row.update(user) + print("### new user info %s" % self.userinfo(row)) + break + self.data.save(self.config_path) - def delete(self, user): - self.data.load(self.config_path) - index = 0 - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - print("delete user [%s]" % row['user']) - del self.data.json[index] - break - index += 1 - self.data.save(self.config_path) + def delete(self, user): + self.data.load(self.config_path) + index = 0 + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("delete user [%s]" % row['user']) + del self.data.json[index] + break + index += 1 + self.data.save(self.config_path) - def clear_ud(self, user): - up = {'u': 0, 'd': 0} - self.data.load(self.config_path) - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - row.update(up) - print("clear user [%s]" % row['user']) - self.data.save(self.config_path) + def clear_ud(self, user): + up = {'u': 0, 'd': 0} + self.data.load(self.config_path) + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + row.update(up) + print("clear user [%s]" % row['user']) + self.data.save(self.config_path) - def list_user(self, user): - self.data.load(self.config_path) - if not user: - for row in self.data.json: - print("user [%s] port %s" % (row['user'], row['port'])) - return - for row in self.data.json: - match = True - if 'user' in user and row['user'] != user['user']: - match = False - if 'port' in user and row['port'] != user['port']: - match = False - if match: - print("### user [%s] info %s" % (row['user'], self.userinfo(row))) + def list_user(self, user): + self.data.load(self.config_path) + if not user: + for row in self.data.json: + print("user [%s] port %s" % (row['user'], row['port'])) + return + for row in self.data.json: + match = True + if 'user' in user and row['user'] != user['user']: + match = False + if 'port' in user and row['port'] != user['port']: + match = False + if match: + print("### user [%s] info %s" % (row['user'], self.userinfo(row))) def print_server_help(): - print('''usage: python mujson_manage.py -a|-d|-e|-c|-l [OPTION]... + print('''usage: python mujson_manage.py -a|-d|-e|-c|-l [OPTION]... Actions: -a ADD add/edit a user @@ -212,118 +207,117 @@ def print_server_help(): def main(): - shortopts = 'adeclu:p:k:O:o:G:g:m:t:f:h' - longopts = ['help'] - action = None - user = {} - fast_set_obfs = {'0': 'plain', - '+1': 'http_simple_compatible', - '1': 'http_simple', - '+2': 'tls1.2_ticket_auth_compatible', - '2': 'tls1.2_ticket_auth'} - fast_set_protocol = {'0': 'origin', - '+1': 'verify_sha1_compatible', - '1': 'verify_sha1', - '2': 'auth_sha1', - '3': 'auth_sha1_v2', - '4': 'auth_sha1_v4', - 'am': 'auth_aes128_md5', - 'as': 'auth_aes128_sha1', - } - fast_set_method = {'a0': 'aes-128-cfb', - 'a1': 'aes-192-cfb', - 'a2': 'aes-256-cfb', - 'r': 'rc4-md5', - 'r6': 'rc4-md5-6', - 'c': 'chacha20', - 'ci': 'chacha20-ietf', - 's': 'salsa20', - 'b': 'bf-cfb', - 'm0': 'camellia-128-cfb', - 'm1': 'camellia-192-cfb', - 'm2': 'camellia-256-cfb', - 'a0t': 'aes-128-ctr', - 'a1t': 'aes-192-ctr', - 'a2t': 'aes-256-ctr'} - try: - optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) - for key, value in optlist: - if key == '-a': - action = 1 - elif key == '-d': - action = 2 - elif key == '-e': - action = 3 - elif key == '-l': - action = 4 - elif key == '-c': - action = 0 - elif key == '-u': - user['user'] = value - elif key == '-p': - user['port'] = int(value) - elif key == '-k': - user['passwd'] = value - elif key == '-o': - if value in fast_set_obfs: - user['obfs'] = fast_set_obfs[value] - else: - user['obfs'] = value - elif key == '-O': - if value in fast_set_protocol: - user['protocol'] = fast_set_protocol[value] - else: - user['protocol'] = value - elif key == '-g': - user['obfs_param'] = value - elif key == '-G': - user['protocol_param'] = value - elif key == '-m': - if value in fast_set_method: - user['method'] = fast_set_method[value] - else: - user['method'] = value - elif key == '-f': - user['forbidden_port'] = value - elif key == '-t': - val = float(value) - try: - val = int(value) - except: - pass - user['transfer_enable'] = val * (1024 ** 3) - elif key in ('-h', '--help'): - print_server_help() - sys.exit(0) - except getopt.GetoptError as e: - print(e) - sys.exit(2) - - manage = MuMgr() - if action == 0: - manage.clear_ud(user) - elif action == 1: - if 'user' not in user and 'port' in user: - user['user'] = str(user['port']) - if 'user' in user and 'port' in user: - manage.add(user) - else: - print("You have to set the port with -p") - elif action == 2: - if 'user' in user or 'port' in user: - manage.delete(user) - else: - print("You have to set the user name or port with -u/-p") - elif action == 3: - if 'user' in user or 'port' in user: - manage.edit(user) - else: - print("You have to set the user name or port with -u/-p") - elif action == 4: - manage.list_user(user) - elif action is None: - print_server_help() + shortopts = 'adeclu:p:k:O:o:G:g:m:t:f:h' + longopts = ['help'] + action = None + user = {} + fast_set_obfs = {'0': 'plain', + '+1': 'http_simple_compatible', + '1': 'http_simple', + '+2': 'tls1.2_ticket_auth_compatible', + '2': 'tls1.2_ticket_auth'} + fast_set_protocol = {'0': 'origin', + '+1': 'verify_sha1_compatible', + '1': 'verify_sha1', + '2': 'auth_sha1', + '3': 'auth_sha1_v2', + '4': 'auth_sha1_v4', + 'am': 'auth_aes128_md5', + 'as': 'auth_aes128_sha1', + } + fast_set_method = {'a0': 'aes-128-cfb', + 'a1': 'aes-192-cfb', + 'a2': 'aes-256-cfb', + 'r': 'rc4-md5', + 'r6': 'rc4-md5-6', + 'c': 'chacha20', + 'ci': 'chacha20-ietf', + 's': 'salsa20', + 'b': 'bf-cfb', + 'm0': 'camellia-128-cfb', + 'm1': 'camellia-192-cfb', + 'm2': 'camellia-256-cfb', + 'a0t': 'aes-128-ctr', + 'a1t': 'aes-192-ctr', + 'a2t': 'aes-256-ctr'} + try: + optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + for key, value in optlist: + if key == '-a': + action = 1 + elif key == '-d': + action = 2 + elif key == '-e': + action = 3 + elif key == '-l': + action = 4 + elif key == '-c': + action = 0 + elif key == '-u': + user['user'] = value + elif key == '-p': + user['port'] = int(value) + elif key == '-k': + user['passwd'] = value + elif key == '-o': + if value in fast_set_obfs: + user['obfs'] = fast_set_obfs[value] + else: + user['obfs'] = value + elif key == '-O': + if value in fast_set_protocol: + user['protocol'] = fast_set_protocol[value] + else: + user['protocol'] = value + elif key == '-g': + user['obfs_param'] = value + elif key == '-G': + user['protocol_param'] = value + elif key == '-m': + if value in fast_set_method: + user['method'] = fast_set_method[value] + else: + user['method'] = value + elif key == '-f': + user['forbidden_port'] = value + elif key == '-t': + val = float(value) + try: + val = int(value) + except: + pass + user['transfer_enable'] = val * (1024 ** 3) + elif key in ('-h', '--help'): + print_server_help() + sys.exit(0) + except getopt.GetoptError as e: + print(e) + sys.exit(2) + manage = MuMgr() + if action == 0: + manage.clear_ud(user) + elif action == 1: + if 'user' not in user and 'port' in user: + user['user'] = str(user['port']) + if 'user' in user and 'port' in user: + manage.add(user) + else: + print("You have to set the port with -p") + elif action == 2: + if 'user' in user or 'port' in user: + manage.delete(user) + else: + print("You have to set the user name or port with -u/-p") + elif action == 3: + if 'user' in user or 'port' in user: + manage.edit(user) + else: + print("You have to set the user name or port with -u/-p") + elif action == 4: + manage.list_user(user) + elif action is None: + print_server_help() if __name__ == '__main__': - main() + main() From e6410c067b59e1976d2e1bd55fe26b376cf59014 Mon Sep 17 00:00:00 2001 From: noisyfox Date: Sat, 7 Jan 2017 21:10:49 +1100 Subject: [PATCH 342/487] Don't clear file first when saving config file --- mujson_mgr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index a20a91fd..ecf473b2 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -27,8 +27,11 @@ def load(self, path): def save(self, path): if self.json: output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) - with open(path, 'wb') as f: + with open(path, 'a'): + pass + with open(path, 'r+') as f: f.write(output.encode('utf8')) + f.truncate() class MuMgr(object): From cd825c9c2c12d443cb732c36960e1f882eb7e492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 7 Jan 2017 22:07:13 +0800 Subject: [PATCH 343/487] typo --- mujson_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index ecf473b2..c01b23d7 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -29,7 +29,7 @@ def save(self, path): output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) with open(path, 'a'): pass - with open(path, 'r+') as f: + with open(path, 'rb+') as f: f.write(output.encode('utf8')) f.truncate() From 4f52dffcc1c1d7aadf60637ad43acada84b90777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 8 Jan 2017 00:26:49 +0800 Subject: [PATCH 344/487] parse protocol_param in py3 --- shadowsocks/obfsplugin/auth.py | 8 ++++---- shadowsocks/tcprelay.py | 8 ++++---- shadowsocks/udprelay.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 4150aaa7..313a1703 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1384,14 +1384,14 @@ def server_udp_pre_encrypt(self, buf): def server_udp_post_decrypt(self, buf): uid = buf[-8:-4] if uid in self.server_info.users: - self.user_key = self.hashfunc(self.server_info.users[uid]).digest() + user_key = self.hashfunc(self.server_info.users[uid]).digest() else: uid = None if not self.server_info.users: - self.user_key = self.server_info.key + user_key = self.server_info.key else: - self.user_key = self.server_info.recv_iv - if hmac.new(self.user_key, buf[:-4], self.hashfunc).digest()[:4] != buf[-4:]: + user_key = self.server_info.recv_iv + if hmac.new(user_key, buf[:-4], self.hashfunc).digest()[:4] != buf[-4:]: return (b'', None) return (buf[:-8], uid) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8db91f9a..83a52742 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1019,13 +1019,13 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun listen_port = config['server_port'] self._listen_port = listen_port - if config['protocol'] in ["auth_aes128_md5", "auth_aes128_sha1"]: - param = config['protocol_param'].split('#') + if common.to_bytes(config['protocol']) in ["auth_aes128_md5", "auth_aes128_sha1"]: + param = common.to_bytes(config['protocol_param']).split(b'#') if len(param) == 2: - user_list = param[1].split(',') + user_list = param[1].split(b',') if user_list: for user in user_list: - items = user.split(':') + items = user.split(b':') if len(items) == 2: uid = struct.pack(' Date: Sun, 8 Jan 2017 00:47:44 +0800 Subject: [PATCH 345/487] typo --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 83a52742..51d1df9c 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1019,7 +1019,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun listen_port = config['server_port'] self._listen_port = listen_port - if common.to_bytes(config['protocol']) in ["auth_aes128_md5", "auth_aes128_sha1"]: + if common.to_bytes(config['protocol']) in [b"auth_aes128_md5", b"auth_aes128_sha1"]: param = common.to_bytes(config['protocol_param']).split(b'#') if len(param) == 2: user_list = param[1].split(b',') From 25c66dab14ab1312cca70c27a513662308ce19c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 8 Jan 2017 17:06:08 +0800 Subject: [PATCH 346/487] bump --- CHANGES | 3 +++ shadowsocks/version.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index ee2741e7..db0ae4bb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +3.0.4 2017-01-08 +- multi-user in single port + 3.0.1 2017-01-03 - remove auth_aes128_*_compatible diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 6bab5d7f..db6c4a10 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- # -# Copyright 2016 breakwa11 +# Copyright 2017 breakwa11 # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.0.1 2017-01-03' + return '3.0.2 2017-01-08' From 278ce30c66051141858b2662f5e6fe974223581d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 9 Jan 2017 17:48:19 +0800 Subject: [PATCH 347/487] update mu online client manager --- shadowsocks/obfsplugin/auth.py | 55 ++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 313a1703..c8057966 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1106,6 +1106,53 @@ def server_udp_post_decrypt(self, buf): return (b'', None) return (data, None) +class obfs_auth_mu_data(object): + def __init__(self): + self.user_id = {} + self.local_client_id = b'' + self.connection_id = 0 + self.set_max_client(64) # max active client count + + def update(self, user_id, client_id, connection_id): + if user_id not in self.user_id: + self.user_id[user_id] = lru_cache.LRUCache() + local_client_id = self.user_id[user_id] + + if client_id in local_client_id: + local_client_id[client_id].update() + + def set_max_client(self, max_client): + self.max_client = max_client + self.max_buffer = max(self.max_client * 2, 1024) + + def insert(self, user_id, client_id, connection_id): + if user_id not in self.user_id: + self.user_id[user_id] = lru_cache.LRUCache() + local_client_id = self.user_id[user_id] + + if local_client_id.get(client_id, None) is None or not local_client_id[client_id].enable: + if local_client_id.first() is None or len(local_client_id) < self.max_client: + if client_id not in local_client_id: + #TODO: check + local_client_id[client_id] = client_queue(connection_id) + else: + local_client_id[client_id].re_enable(connection_id) + return local_client_id[client_id].insert(connection_id) + + if not local_client_id[local_client_id.first()].is_active(): + del local_client_id[local_client_id.first()] + if client_id not in local_client_id: + #TODO: check + local_client_id[client_id] = client_queue(connection_id) + else: + local_client_id[client_id].re_enable(connection_id) + return local_client_id[client_id].insert(connection_id) + + logging.warn('auth_aes128: no inactive client') + return False + else: + return local_client_id[client_id].insert(connection_id) + class auth_aes128_sha1(auth_base): def __init__(self, method, hashfunc): super(auth_aes128_sha1, self).__init__(method) @@ -1123,10 +1170,11 @@ def __init__(self, method, hashfunc): self.extra_wait_size = struct.unpack('>H', os.urandom(2))[0] % 1024 self.pack_id = 1 self.recv_id = 1 + self.user_id = None self.user_key = None def init_data(self): - return obfs_auth_v2_data() + return obfs_auth_mu_data() def set_server_info(self, server_info): self.server_info = server_info @@ -1282,6 +1330,7 @@ def server_post_decrypt(self, buf): uid = self.recv_buf[7:11] if uid in self.server_info.users: + self.user_id = uid self.user_key = self.hashfunc(self.server_info.users[uid]).digest() self.server_info.update_user_func(uid) else: @@ -1306,7 +1355,7 @@ def server_post_decrypt(self, buf): if time_dif < -self.max_time_dif or time_dif > self.max_time_dif: logging.info('%s: wrong timestamp, time_dif %d, data %s' % (self.no_compatible_method, time_dif, binascii.hexlify(head))) return self.not_match_return(self.recv_buf) - elif self.server_info.data.insert(client_id, connection_id): + elif self.server_info.data.insert(self.user_id, client_id, connection_id): self.has_recv_header = True out_buf = self.recv_buf[31 + rnd_len:length - 4] self.client_id = client_id @@ -1362,7 +1411,7 @@ def server_post_decrypt(self, buf): sendback = True if out_buf: - self.server_info.data.update(self.client_id, self.connection_id) + self.server_info.data.update(self.user_id, self.client_id, self.connection_id) return (out_buf, sendback) def client_udp_pre_encrypt(self, buf): From 588d8ad8b320526c575705cf029b24803fac9db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 11 Jan 2017 14:43:24 +0800 Subject: [PATCH 348/487] fix auth_aes128_* client side --- shadowsocks/obfsplugin/auth.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index c8057966..29730f3b 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1222,9 +1222,16 @@ def pack_auth_data(self, auth_data, buf): data = data + struct.pack(' Date: Wed, 11 Jan 2017 14:45:37 +0800 Subject: [PATCH 349/487] multi-user for mudbjson mode --- db_transfer.py | 67 +++++++++++++++++++++++++++------------- mujson_mgr.py | 6 +++- server_pool.py | 68 ++++++++++++++++++++++++++++++++++++----- shadowsocks/tcprelay.py | 51 ++++++++++++++++++++++++------- shadowsocks/udprelay.py | 49 ++++++++++++++++++++++------- 5 files changed, 188 insertions(+), 53 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 2ef5829c..ec9152b2 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -85,6 +85,8 @@ def del_server_out_of_bound_safe(self, last_rows, rows): logging.error('load switchrule.py fail') cur_servers = {} new_servers = {} + allow_users = {} + mu_servers = {} for row in rows: try: allow = switchrule.isTurnOn(row) and row['enable'] == 1 and row['u'] + row['d'] < row['transfer_enable'] @@ -113,34 +115,50 @@ def del_server_out_of_bound_safe(self, last_rows, rows): logging.error('more than one user use the same port [%s]' % (port,)) continue - if ServerPool.get_instance().server_is_run(port) > 0: - if not allow: - logging.info('db stop server at port [%s]' % (port,)) - ServerPool.get_instance().cb_del_server(port) - self.force_update_transfer.add(port) - else: - cfgchange = False - if port in ServerPool.get_instance().tcp_servers_pool: - relay = ServerPool.get_instance().tcp_servers_pool[port] - for name in merge_config_keys: - if name in cfg and not self.cmp(cfg[name], relay._config[name]): - cfgchange = True - break; - if not cfgchange and port in ServerPool.get_instance().tcp_ipv6_servers_pool: - relay = ServerPool.get_instance().tcp_ipv6_servers_pool[port] - for name in merge_config_keys: - if name in cfg and not self.cmp(cfg[name], relay._config[name]): - cfgchange = True - break; - #config changed + if allow: + allow_users[port] = 1 + if 'protocol' in cfg and 'protocol_param' in cfg and common.to_str(cfg['protocol']) in ['auth_aes128_md5', 'auth_aes128_sha1']: + if '#' in common.to_str(cfg['protocol_param']): + mu_servers[port] = 1 + + cfgchange = False + if port in ServerPool.get_instance().tcp_servers_pool: + relay = ServerPool.get_instance().tcp_servers_pool[port] + for name in merge_config_keys: + if name in cfg and not self.cmp(cfg[name], relay._config[name]): + cfgchange = True + break + if not cfgchange and port in ServerPool.get_instance().tcp_ipv6_servers_pool: + relay = ServerPool.get_instance().tcp_ipv6_servers_pool[port] + for name in merge_config_keys: + if name in cfg and not self.cmp(cfg[name], relay._config[name]): + cfgchange = True + break + + if port in mu_servers: + if ServerPool.get_instance().server_is_run(port) > 0: if cfgchange: logging.info('db stop server at port [%s] reason: config changed: %s' % (port, cfg)) ServerPool.get_instance().cb_del_server(port) self.force_update_transfer.add(port) new_servers[port] = (passwd, cfg) + else: + self.new_server(port, passwd, cfg) + else: + if ServerPool.get_instance().server_is_run(port) > 0: + if not allow: + logging.info('db stop server at port [%s]' % (port,)) + ServerPool.get_instance().cb_del_server(port) + self.force_update_transfer.add(port) + else: + if cfgchange: + logging.info('db stop server at port [%s] reason: config changed: %s' % (port, cfg)) + ServerPool.get_instance().cb_del_server(port) + self.force_update_transfer.add(port) + new_servers[port] = (passwd, cfg) - elif allow and ServerPool.get_instance().server_run_status(port) is False: - self.new_server(port, passwd, cfg) + elif allow and port > 0 and port < 65536 and ServerPool.get_instance().server_run_status(port) is False: + self.new_server(port, passwd, cfg) for row in last_rows: if row['port'] in cur_servers: @@ -159,6 +177,11 @@ def del_server_out_of_bound_safe(self, last_rows, rows): passwd, cfg = new_servers[port] self.new_server(port, passwd, cfg) + if isinstance(self, MuJsonTransfer): # works in MuJsonTransfer only + logging.debug('db allow users %s \nmu_servers %s' % (allow_users, mu_servers)) + for port in mu_servers: + ServerPool.get_instance().update_mu_server(port, None, allow_users) + def clear_cache(self, port): if port in self.force_update_transfer: del self.force_update_transfer[port] if port in self.last_get_transfer: del self.last_get_transfer[port] diff --git a/mujson_mgr.py b/mujson_mgr.py index c01b23d7..a880a1c8 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -73,8 +73,12 @@ def ssrlink(self, user, encode): def userinfo(self, user): ret = "" + key_list = ['user', 'port', 'method', 'passwd', 'protocol', 'protocol_param', 'obfs', 'obfs_param', 'transfer_enable', 'u', 'd'] for key in sorted(user): - if key in ['enable']: + if key not in key_list: + key_list.append(key) + for key in key_list: + if key in ['enable'] or key not in user: continue ret += '\n' if key in ['transfer_enable', 'u', 'd']: diff --git a/server_pool.py b/server_pool.py index a526bd05..45fe6a13 100644 --- a/server_pool.py +++ b/server_pool.py @@ -23,6 +23,7 @@ import os import logging +import struct import time from shadowsocks import shell, eventloop, tcprelay, udprelay, asyncdns, common import threading @@ -213,23 +214,62 @@ def cb_del_server(self, port): return True + def update_mu_server(self, port, protocol_param, acl): + port = int(port) + if port in self.tcp_servers_pool: + try: + self.tcp_servers_pool[port].update_users(protocol_param, acl) + except Exception as e: + logging.warn(e) + try: + self.udp_servers_pool[port].update_users(protocol_param, acl) + except Exception as e: + logging.warn(e) + if port in self.tcp_ipv6_servers_pool: + try: + self.tcp_ipv6_servers_pool[port].update_users(protocol_param, acl) + except Exception as e: + logging.warn(e) + try: + self.udp_ipv6_servers_pool[port].update_users(protocol_param, acl) + except Exception as e: + logging.warn(e) + def get_server_transfer(self, port): port = int(port) + uid = struct.pack(' Date: Wed, 11 Jan 2017 18:34:28 +0800 Subject: [PATCH 350/487] let 'transfer_enable' be an integer --- mujson_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index a880a1c8..312bb0ee 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -293,7 +293,7 @@ def main(): val = int(value) except: pass - user['transfer_enable'] = val * (1024 ** 3) + user['transfer_enable'] = int(val * 1024) * (1024 ** 2) elif key in ('-h', '--help'): print_server_help() sys.exit(0) From d4b7483ef0c62d7182891d1b6bb706724f89008b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 12 Jan 2017 22:06:20 +0800 Subject: [PATCH 351/487] modify mgr fast command --- mujson_mgr.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 312bb0ee..23338ef3 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -224,11 +224,14 @@ def main(): '+2': 'tls1.2_ticket_auth_compatible', '2': 'tls1.2_ticket_auth'} fast_set_protocol = {'0': 'origin', - '+1': 'verify_sha1_compatible', - '1': 'verify_sha1', - '2': 'auth_sha1', - '3': 'auth_sha1_v2', - '4': 'auth_sha1_v4', + '+ota': 'verify_sha1_compatible', + 'ota': 'verify_sha1', + 'a1': 'auth_sha1', + '+a1': 'auth_sha1_compatible', + 'a2': 'auth_sha1_v2', + '+a2': 'auth_sha1_v2_compatible', + 'a4': 'auth_sha1_v4', + '+a4': 'auth_sha1_v4_compatible', 'am': 'auth_aes128_md5', 'as': 'auth_aes128_sha1', } From 4048abaeff828fd47ea8b6f65324d8e384f19257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 13 Jan 2017 21:52:17 +0800 Subject: [PATCH 352/487] refine user transfer --- shadowsocks/tcprelay.py | 6 ++++-- shadowsocks/udprelay.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index abbb9669..da0565c7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1108,7 +1108,8 @@ def add_transfer_u(self, user, transfer): else: if user not in self.server_user_transfer_ul: self.server_user_transfer_ul[user] = 0 - self.server_user_transfer_ul[user] += transfer + self.server_user_transfer_ul[user] += transfer + self.server_transfer_ul + self.server_transfer_ul = 0 def add_transfer_d(self, user, transfer): if user is None: @@ -1116,7 +1117,8 @@ def add_transfer_d(self, user, transfer): else: if user not in self.server_user_transfer_dl: self.server_user_transfer_dl[user] = 0 - self.server_user_transfer_dl[user] += transfer + self.server_user_transfer_dl[user] += transfer + self.server_transfer_dl + self.server_transfer_dl = 0 def update_stat(self, port, stat_dict, val): newval = stat_dict.get(0, 0) + val diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 994fdeed..9ee5ccf8 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1008,7 +1008,8 @@ def add_transfer_u(self, user, transfer): else: if user not in self.server_user_transfer_ul: self.server_user_transfer_ul[user] = 0 - self.server_user_transfer_ul[user] += transfer + self.server_user_transfer_ul[user] += transfer + self.server_transfer_ul + self.server_transfer_ul = 0 def add_transfer_d(self, user, transfer): if user is None: @@ -1016,7 +1017,8 @@ def add_transfer_d(self, user, transfer): else: if user not in self.server_user_transfer_dl: self.server_user_transfer_dl[user] = 0 - self.server_user_transfer_dl[user] += transfer + self.server_user_transfer_dl[user] += transfer + self.server_transfer_dl + self.server_transfer_dl = 0 def _close_client_pair(self, client_pair): client, uid = client_pair From 75643635d4e68a0d5127b1aa3aa15fc343b094ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 15 Jan 2017 10:05:29 +0800 Subject: [PATCH 353/487] change default setting --- config.json | 2 +- mujson_mgr.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config.json b/config.json index 199b111c..92faf5ad 100644 --- a/config.json +++ b/config.json @@ -7,7 +7,7 @@ "password": "m", "timeout": 120, "udp_timeout": 60, - "method": "aes-256-cfb", + "method": "aes-128-ctr", "protocol": "auth_aes128_md5", "protocol_param": "", "obfs": "tls1.2_ticket_auth_compatible", diff --git a/mujson_mgr.py b/mujson_mgr.py index 23338ef3..46ab3e17 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -104,8 +104,8 @@ def rand_pass(self): return ''.join([random.choice('''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~-_=+(){}[]^&%$@''') for i in range(8)]) def add(self, user): - up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-cfb", - 'protocol': "auth_sha1_v4", + up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-ctr", + 'protocol': "auth_aes128_md5", 'obfs': "tls1.2_ticket_auth_compatible", 'transfer_enable': 1125899906842624} up['passwd'] = self.rand_pass() From ccacef155611a1cd87299b86a76311a2551e88ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 17 Jan 2017 20:53:54 +0800 Subject: [PATCH 354/487] fix bug & update usage --- mujson_mgr.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 46ab3e17..88814317 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -25,7 +25,7 @@ def load(self, path): self.json = json.loads(l) def save(self, path): - if self.json: + if self.json is not None: output = json.dumps(self.json, sort_keys=True, indent=4, separators=(',', ': ')) with open(path, 'a'): pass @@ -107,7 +107,7 @@ def add(self, user): up = {'enable': 1, 'u': 0, 'd': 0, 'method': "aes-128-ctr", 'protocol': "auth_aes128_md5", 'obfs': "tls1.2_ticket_auth_compatible", - 'transfer_enable': 1125899906842624} + 'transfer_enable': 9007199254740992} up['passwd'] = self.rand_pass() up.update(user) @@ -190,26 +190,26 @@ def print_server_help(): print('''usage: python mujson_manage.py -a|-d|-e|-c|-l [OPTION]... Actions: - -a ADD add/edit a user - -d DELETE delete a user - -e EDIT edit a user - -c CLEAR set u/d to zero - -l LIST display a user infomation or all users infomation + -a ADD add/edit a user + -d DELETE delete a user + -e EDIT edit a user + -c CLEAR set u/d to zero + -l LIST display a user infomation or all users infomation Options: - -u USER the user name - -p PORT server port - -k PASSWORD password - -m METHOD encryption method, default: aes-128-cfb - -O PROTOCOL protocol plugin, default: auth_sha1_v4 - -o OBFS obfs plugin, default: tls1.2_ticket_auth_compatible - -G PROTOCOL_PARAM protocol plugin param - -g OBFS_PARAM obfs plugin param - -t TRANSFER max transfer for G bytes, default: 1048576, can be float point number - -f FORBID set forbidden ports. Example (ban 1~79 and 81~100): -f "1-79,81-100" + -u USER the user name + -p PORT server port (only this option must be set if add a user) + -k PASSWORD password + -m METHOD encryption method, default: aes-128-ctr + -O PROTOCOL protocol plugin, default: auth_aes128_md5 + -o OBFS obfs plugin, default: tls1.2_ticket_auth_compatible + -G PROTOCOL_PARAM protocol plugin param + -g OBFS_PARAM obfs plugin param + -t TRANSFER max transfer for G bytes, default: 8388608 (8 PB or 8192 TB) + -f FORBID set forbidden ports. Example (ban 1~79 and 81~100): -f "1-79,81-100" General options: - -h, --help show this help message and exit + -h, --help show this help message and exit ''') From 9c794624e0e6f577d7198de4fbe7de69a349448e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 18 Jan 2017 17:00:09 +0800 Subject: [PATCH 355/487] fix auth_aes128_* client side --- shadowsocks/obfsplugin/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 29730f3b..b1adc372 100644 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1224,7 +1224,7 @@ def pack_auth_data(self, auth_data, buf): uid = os.urandom(4) if b':' in to_bytes(self.server_info.protocol_param): try: - items = to_bytes(self.server_info.protocol_param).split(':') + items = to_bytes(self.server_info.protocol_param).split(b':') self.user_key = self.hashfunc(items[1]).digest() uid = struct.pack(' Date: Wed, 18 Jan 2017 07:05:15 -0600 Subject: [PATCH 356/487] refine traffic log when redirect enable (#163) --- shadowsocks/tcprelay.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index da0565c7..28eed260 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -313,7 +313,7 @@ def _write_to_sock(self, data, sock): if self._encrypt_correct: if sock == self._remote_sock: self._server.add_transfer_u(self._user, len(data)) - self._update_activity(len(data)) + self._update_activity(len(data)) if data: l = len(data) s = sock.send(data) @@ -847,8 +847,8 @@ def _on_remote_read(self, is_remote_sock): data = self._protocol.server_pre_encrypt(data) data = self._encryptor.encrypt(data) data = self._obfs.server_encode(data) - self._update_activity(len(data)) - self._server.add_transfer_d(self._user, len(data)) + self._server.add_transfer_d(self._user, len(data)) + self._update_activity(len(data)) else: return try: From 68693d5ec7475cf265684a68521562501584c995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 21 Jan 2017 12:43:22 +0800 Subject: [PATCH 357/487] fix bug under py3 --- db_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index ec9152b2..fa10a4bc 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -278,7 +278,7 @@ def load_cfg(self): import json config_path = get_config().MYSQL_CONFIG cfg = None - with open(config_path, 'r+') as f: + with open(config_path, 'rb+') as f: cfg = json.loads(f.read().decode('utf8')) if cfg: From 00a31285ed3abf5464cf34ff0bdc8723b88bb303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 22 Jan 2017 13:54:56 +0800 Subject: [PATCH 358/487] encode protocol param to URL with -i --- mujson_mgr.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 88814317..5d85798c 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -63,15 +63,32 @@ def getipaddr(self, ifname='eth0'): pass return ret - def ssrlink(self, user, encode): + def ssrlink(self, user, encode, muid): protocol = user.get('protocol', '') obfs = user.get('obfs', '') protocol = protocol.replace("_compatible", "") obfs = obfs.replace("_compatible", "") link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace("=", "")) + if muid is not None: + protocol_param = user.get('protocol_param', '') + param = protocol_param.split('#') + if len(param) == 2: + user_dict = {} + user_list = param[1].split(',') + if user_list: + for userinfo in user_list: + items = userinfo.split(':') + if len(items) == 2: + user_int_id = int(items[0]) + passwd = items[1] + user_dict[user_int_id] = passwd + if muid in user_dict: + param = str(muid) + ':' + user_dict[muid] + protocol_param = '/?protoparam=' + base64.urlsafe_b64encode(common.to_bytes(param)).replace("=", "") + link += protocol_param return "ssr://" + (encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) - def userinfo(self, user): + def userinfo(self, user, muid = None): ret = "" key_list = ['user', 'port', 'method', 'passwd', 'protocol', 'protocol_param', 'obfs', 'obfs_param', 'transfer_enable', 'u', 'd'] for key in sorted(user): @@ -96,8 +113,8 @@ def userinfo(self, user): ret += " %s : %s G Bytes" % (key, val) else: ret += " %s : %s" % (key, user[key]) - ret += "\n " + self.ssrlink(user, False) - ret += "\n " + self.ssrlink(user, True) + ret += "\n " + self.ssrlink(user, False, muid) + ret += "\n " + self.ssrlink(user, True, muid) return ret def rand_pass(self): @@ -183,7 +200,10 @@ def list_user(self, user): if 'port' in user and row['port'] != user['port']: match = False if match: - print("### user [%s] info %s" % (row['user'], self.userinfo(row))) + muid = None + if 'muid' in user: + muid = user['muid'] + print("### user [%s] info %s" % (row['user'], self.userinfo(row, muid))) def print_server_help(): @@ -207,6 +227,7 @@ def print_server_help(): -g OBFS_PARAM obfs plugin param -t TRANSFER max transfer for G bytes, default: 8388608 (8 PB or 8192 TB) -f FORBID set forbidden ports. Example (ban 1~79 and 81~100): -f "1-79,81-100" + -i MUID set sub id to display (only work with -l) General options: -h, --help show this help message and exit @@ -214,7 +235,7 @@ def print_server_help(): def main(): - shortopts = 'adeclu:p:k:O:o:G:g:m:t:f:h' + shortopts = 'adeclu:i:p:k:O:o:G:g:m:t:f:h' longopts = ['help'] action = None user = {} @@ -265,6 +286,8 @@ def main(): action = 0 elif key == '-u': user['user'] = value + elif key == '-i': + user['muid'] = int(value) elif key == '-p': user['port'] = int(value) elif key == '-k': From 96760ed4384ed3755c185e065f4e5c79e80f6028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 23 Jan 2017 16:28:59 +0800 Subject: [PATCH 359/487] not display RST error --- shadowsocks/tcprelay.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 28eed260..571fe42e 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -880,20 +880,22 @@ def _on_remote_write(self): self._update_stream(STREAM_UP, WAIT_STATUS_READING) def _on_local_error(self): - logging.debug('got local error') if self._local_sock: - logging.error(eventloop.get_sock_error(self._local_sock)) - logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) + err = eventloop.get_sock_error(self._local_sock) + if err.errno not in [errno.ECONNRESET]: + logging.error(err) + logging.error("local error, exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() def _on_remote_error(self): - logging.debug('got remote error') if self._remote_sock: - logging.error(eventloop.get_sock_error(self._remote_sock)) - if self._remote_address: - logging.error("when connect to %s:%d" % (self._remote_address[0], self._remote_address[1])) - else: - logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) + err = eventloop.get_sock_error(self._remote_sock) + if err.errno not in [errno.ECONNRESET]: + logging.error(err) + if self._remote_address: + logging.error("remote error, when connect to %s:%d" % (self._remote_address[0], self._remote_address[1])) + else: + logging.error("remote error, exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() def handle_event(self, sock, event): From e7a61a934fe8db4c5842b73feb36e9340a8a8bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 25 Jan 2017 01:08:00 +0800 Subject: [PATCH 360/487] ignore SIGPIPE --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 571fe42e..dbcba089 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -882,7 +882,7 @@ def _on_remote_write(self): def _on_local_error(self): if self._local_sock: err = eventloop.get_sock_error(self._local_sock) - if err.errno not in [errno.ECONNRESET]: + if err.errno not in [errno.ECONNRESET, errno.EPIPE]: logging.error(err) logging.error("local error, exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() From 390be3bf8d34e195b20bb7b0dcfcc94915f73078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 2 Feb 2017 19:05:18 +0800 Subject: [PATCH 361/487] random ticket size for client only --- shadowsocks/obfsplugin/obfs_tls.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 7646f6e8..3416c09d 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -54,6 +54,7 @@ def __init__(self): self.client_data = lru_cache.LRUCache(60 * 5) self.client_id = os.urandom(32) self.startup_time = int(time.time() - 60 * 30) & 0xFFFFFFFF + self.ticket_buf = {} class tls_ticket_auth(plain.plain): def __init__(self, method): @@ -104,7 +105,9 @@ def client_encode(self, buf): host = random.choice(hosts) ext += self.sni(host) ext += b"\x00\x17\x00\x00" - ext += b"\x00\x23\x00\xd0" + os.urandom(208) # ticket + if host not in self.server_info.data.ticket_buf: + self.server_info.data.ticket_buf[host] = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 17 + 8) * 16) + ext += b"\x00\x23" + struct.pack('>H', len(self.ticket_size_buf[host])) + self.ticket_size_buf[host] ext += binascii.unhexlify(b"000d001600140601060305010503040104030301030302010203") ext += binascii.unhexlify(b"000500050100000000") ext += binascii.unhexlify(b"00120000") From 31aaea603a5943c43e5f26b9c966c97c2d51aabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 2 Feb 2017 19:50:55 +0800 Subject: [PATCH 362/487] add "glzjinmod" interface --- apiconfig.py | 2 +- db_transfer.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apiconfig.py b/apiconfig.py index 58ea33a2..cbeba7f0 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,5 +1,5 @@ # Config -API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, muapiv2(not support) +API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, glzjinmod, muapiv2(not support) UPDATE_TIME = 60 SERVER_PUB_ADDR = '127.0.0.1' # mujson_mgr need this to generate ssr link diff --git a/db_transfer.py b/db_transfer.py index fa10a4bc..fc9b0f57 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -391,8 +391,11 @@ class Dbv3Transfer(DbTransfer): def __init__(self): super(Dbv3Transfer, self).__init__() self.key_list += ['id', 'method'] + self.ss_node_info_name = 'ss_node_info_log' if get_config().API_INTERFACE == 'sspanelv3ssr': self.key_list += ['obfs', 'protocol'] + if get_config().API_INTERFACE == 'glzjinmod': + self.ss_node_info_name = 'ss_node_info' self.start_time = time.time() def update_all_user(self, dt_transfer): @@ -473,14 +476,14 @@ def update_all_user(self, dt_transfer): cur = conn.cursor() try: - cur.execute("INSERT INTO `ss_node_info_log` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ + cur.execute("INSERT INTO `" + self.ss_node_info_name + "` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ str(self.cfg["node_id"]) + "', '" + str(self.uptime()) + "', '" + \ str(self.load()) + "', unix_timestamp()); ") except Exception as e: logging.error(e) cur.close() except: - logging.warn('no `ss_node_online_log` or `ss_node_info_log` in db') + logging.warn('no `ss_node_online_log` or `" + self.ss_node_info_name + "` in db') conn.close() return update_transfer From bf443d72ecfe00c6010f234e6ad7e0b09e32cfc7 Mon Sep 17 00:00:00 2001 From: Esdeath Date: Sat, 4 Feb 2017 13:01:18 +0800 Subject: [PATCH 363/487] refine add "glzjinmod" interface glzjinmod support these key --- db_transfer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/db_transfer.py b/db_transfer.py index fc9b0f57..4d6088f9 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -395,6 +395,7 @@ def __init__(self): if get_config().API_INTERFACE == 'sspanelv3ssr': self.key_list += ['obfs', 'protocol'] if get_config().API_INTERFACE == 'glzjinmod': + self.key_list += ['obfs', 'protocol'] self.ss_node_info_name = 'ss_node_info' self.start_time = time.time() From 3014b0034c7a530055452587dd334b2ae8e06147 Mon Sep 17 00:00:00 2001 From: Dwwwwww Date: Sun, 5 Feb 2017 12:36:45 +0800 Subject: [PATCH 364/487] Update README.md (#169) Improve --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 14ad3e25..53eafeaa 100644 --- a/README.md +++ b/README.md @@ -33,11 +33,11 @@ move to "~/shadowsocksr", then run: move to "~/shadowsocksr/shadowsocks", then run: - python server.py -p 443 -k password -m aes-128-cfb -O auth_aes128_md5_compatible -o tls1.2_ticket_auth_compatible + python server.py -p 443 -k password -m aes-128-cfb -O auth_aes128_md5 -o tls1.2_ticket_auth_compatible Check all the options via `-h`. -You can also use a configuration file instead (recommand), move to "~/shadowsocksr" and edit the file "user-config.json", then move to "~/shadowsocksr/shadowsocks" again, just run: +You can also use a configuration file instead (recommend), move to "~/shadowsocksr" and edit the file "user-config.json", then move to "~/shadowsocksr/shadowsocks" again, just run: python server.py @@ -57,7 +57,7 @@ To monitor the log: Client ------ -* [Windows] / [OS X] +* [Windows] / [macOS] * [Android] / [iOS] * [OpenWRT] @@ -99,7 +99,7 @@ Bugs and Issues [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help [Issue Tracker]: https://github.com/shadowsocksr/shadowsocksr/issues?state=open [OpenWRT]: https://github.com/shadowsocks/openwrt-shadowsocks -[OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help +[macOS]: https://github.com/shadowsocksr/ShadowsocksX-NG [Travis CI]: https://travis-ci.org/shadowsocksr/shadowsocksr [Windows]: https://github.com/shadowsocksr/shadowsocksr-csharp -[Wiki]: https://github.com/breakwa11/shadowsocks-rss/wiki \ No newline at end of file +[Wiki]: https://github.com/breakwa11/shadowsocks-rss/wiki From 5d479087a88647e9bcc0e2ea900b4903f5dedc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 5 Feb 2017 19:18:50 +0800 Subject: [PATCH 365/487] fix tls1.2_ticket_auth when it as a client --- shadowsocks/obfsplugin/obfs_tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 3416c09d..18839106 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -107,7 +107,7 @@ def client_encode(self, buf): ext += b"\x00\x17\x00\x00" if host not in self.server_info.data.ticket_buf: self.server_info.data.ticket_buf[host] = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 17 + 8) * 16) - ext += b"\x00\x23" + struct.pack('>H', len(self.ticket_size_buf[host])) + self.ticket_size_buf[host] + ext += b"\x00\x23" + struct.pack('>H', len(self.server_info.data.ticket_buf[host])) + self.server_info.data.ticket_buf[host] ext += binascii.unhexlify(b"000d001600140601060305010503040104030301030302010203") ext += binascii.unhexlify(b"000500050100000000") ext += binascii.unhexlify(b"00120000") From bf8bb8afe27dcde8b124c3b849d77474229d5d71 Mon Sep 17 00:00:00 2001 From: John Date: Sun, 5 Feb 2017 23:44:43 +1100 Subject: [PATCH 366/487] fix undefined variable bug (#171) --- shadowsocks/obfsplugin/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 shadowsocks/obfsplugin/auth.py diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py old mode 100644 new mode 100755 index b1adc372..b787b33a --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1434,7 +1434,7 @@ def client_udp_pre_encrypt(self, buf): self.user_id = os.urandom(4) self.user_key = self.server_info.key buf += self.user_id - return buf + hmac.new(user_key, buf, self.hashfunc).digest()[:4] + return buf + hmac.new(self.user_key, buf, self.hashfunc).digest()[:4] def client_udp_post_decrypt(self, buf): user_key = self.server_info.key From 2750af4a2eca25c470b279febc7c2c0698ac082a Mon Sep 17 00:00:00 2001 From: John Date: Sun, 12 Feb 2017 01:56:02 +1100 Subject: [PATCH 367/487] optimize shell (#173) --- shadowsocks/shell.py | 88 ++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 48 deletions(-) mode change 100644 => 100755 shadowsocks/shell.py diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py old mode 100644 new mode 100755 index 755263db..2fb268a6 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -52,8 +52,7 @@ def print_exception(e): import traceback traceback.print_exc() - -def print_shadowsocks(): +def __version(): version_str = '' try: import pkg_resources @@ -64,37 +63,26 @@ def print_shadowsocks(): version_str = version.version() except Exception: pass - print('ShadowsocksR %s' % version_str) + return version_str + +def print_shadowsocks(): + print('ShadowsocksR %s' % __version()) def log_shadowsocks_version(): - version_str = '' - try: - import pkg_resources - version_str = pkg_resources.get_distribution('shadowsocks').version - except Exception: - try: - from shadowsocks import version - version_str = version.version() - except Exception: - pass - logging.info('ShadowsocksR %s' % version_str) + logging.info('ShadowsocksR %s' % __version()) -def find_config(): - config_path = 'user-config.json' - if os.path.exists(config_path): - return config_path - config_path = os.path.join(os.path.dirname(__file__), '../', 'user-config.json') - if os.path.exists(config_path): - return config_path +def find_config(): + user_config_path = 'user-config.json' config_path = 'config.json' - if os.path.exists(config_path): - return config_path - config_path = os.path.join(os.path.dirname(__file__), '../', 'config.json') - if os.path.exists(config_path): - return config_path - return None + def sub_find(file_name): + if os.path.exists(file_name): + return file_name + file_name = os.path.join(os.path.abspath('..'), file_name) + return file_name if os.path.exists(file_name) else None + + return sub_find(user_config_path) or sub_find(config_path) def check_config(config, is_local): if config.get('daemon', None) == 'stop': @@ -119,21 +107,21 @@ def check_config(config, is_local): config['server_port'] = int(config['server_port']) if config.get('local_address', '') in [b'0.0.0.0']: - logging.warn('warning: local set to listen on 0.0.0.0, it\'s not safe') + logging.warning('warning: local set to listen on 0.0.0.0, it\'s not safe') if config.get('server', '') in ['127.0.0.1', 'localhost']: - logging.warn('warning: server set to listen on %s:%s, are you sure?' % + logging.warning('warning: server set to listen on %s:%s, are you sure?' % (to_str(config['server']), config['server_port'])) if (config.get('method', '') or '').lower() == 'table': - logging.warn('warning: table is not safe; please use a safer cipher, ' + logging.warning('warning: table is not safe; please use a safer cipher, ' 'like AES-256-CFB') if (config.get('method', '') or '').lower() == 'rc4': - logging.warn('warning: RC4 is not safe; please use a safer cipher, ' + logging.warning('warning: RC4 is not safe; please use a safer cipher, ' 'like AES-256-CFB') if config.get('timeout', 300) < 100: - logging.warn('warning: your timeout %d seems too short' % + logging.warning('warning: your timeout %d seems too short' % int(config.get('timeout'))) if config.get('timeout', 300) > 600: - logging.warn('warning: your timeout %d seems too long' % + logging.warning('warning: your timeout %d seems too long' % int(config.get('timeout'))) if config.get('password') in [b'mypassword']: logging.error('DON\'T USE DEFAULT PASSWORD! Please change it in your ' @@ -149,7 +137,8 @@ def check_config(config, is_local): def get_config(is_local): global verbose - + config = {} + config_path = None logging.basicConfig(level=logging.INFO, format='%(levelname)-s: %(message)s') if is_local: @@ -161,11 +150,22 @@ def get_config(is_local): longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'workers=', 'forbidden-ip=', 'user=', 'manager-address=', 'version'] try: - config_path = find_config() optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) for key, value in optlist: if key == '-c': config_path = value + elif key in ('-h', '--help'): + print_help(is_local) + sys.exit(0) + elif key == '--version': + print_shadowsocks() + sys.exit(0) + else: + continue + + if config_path is None: + config_path = find_config() + if config_path: logging.info('loading config from %s' % config_path) @@ -173,11 +173,9 @@ def get_config(is_local): try: config = parse_json_in_str(remove_comment(f.read().decode('utf8'))) except ValueError as e: - logging.error('found an error in config.json: %s', - e.message) + logging.error('found an error in config.json: %s', str(e)) sys.exit(1) - else: - config = {} + v_count = 0 for key, value in optlist: @@ -217,15 +215,7 @@ def get_config(is_local): config['user'] = to_str(value) elif key == '--forbidden-ip': config['forbidden_ip'] = to_str(value) - elif key in ('-h', '--help'): - if is_local: - print_local_help() - else: - print_server_help() - sys.exit(0) - elif key == '--version': - print_shadowsocks() - sys.exit(0) + elif key == '-d': config['daemon'] = to_str(value) elif key == '--pid-file': @@ -235,6 +225,8 @@ def get_config(is_local): elif key == '-q': v_count -= 1 config['verbose'] = v_count + else: + continue except getopt.GetoptError as e: print(e, file=sys.stderr) print_help(is_local) From 72ca31dad9cf4cc8cf50dd3d29fc76bf406700e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 12 Feb 2017 22:35:26 +0800 Subject: [PATCH 368/487] add traffic control for each TCP connection --- config.json | 3 ++ shadowsocks/eventloop.py | 5 +++- shadowsocks/lru_cache.py | 4 +-- shadowsocks/tcprelay.py | 62 +++++++++++++++++++++++++++++++++------- shadowsocks/udprelay.py | 19 ++++++++---- 5 files changed, 75 insertions(+), 18 deletions(-) diff --git a/config.json b/config.json index 92faf5ad..0d0d13ea 100644 --- a/config.json +++ b/config.json @@ -4,6 +4,7 @@ "server_port": 8388, "local_address": "127.0.0.1", "local_port": 1080, + "password": "m", "timeout": 120, "udp_timeout": 60, @@ -12,6 +13,8 @@ "protocol_param": "", "obfs": "tls1.2_ticket_auth_compatible", "obfs_param": "", + "speed_limit_per_con": 0, + "dns_ipv6": false, "connect_verbose_info": 0, "redirect": "", diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 2ccd7e6c..a9560fa5 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -208,12 +208,13 @@ def run(self): traceback.print_exc() continue + handle = False for sock, fd, event in events: handler = self._fdmap.get(fd, None) if handler is not None: handler = handler[1] try: - handler.handle_event(sock, fd, event) + handle = handle or handler.handle_event(sock, fd, event) except (OSError, IOError) as e: shell.print_exception(e) now = time.time() @@ -221,6 +222,8 @@ def run(self): for callback in self._periodic_callbacks: callback() self._last_time = now + if events and (handle is False): + time.sleep(0.01) def __del__(self): self._impl.close() diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index d748d587..328ef85a 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -88,11 +88,11 @@ def first(self): for key in self._keys_to_last_time: return key - def sweep(self): + def sweep(self, sweep_item_cnt = SWEEP_MAX_ITEMS): # O(n - m) now = time.time() c = 0 - while c < SWEEP_MAX_ITEMS: + while c < sweep_item_cnt: if len(self._keys_to_last_time) == 0: break for key in self._keys_to_last_time: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index dbcba089..431fe9cc 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -28,6 +28,7 @@ import random import platform import threading +from collections import deque from shadowsocks import encrypt, obfs, eventloop, shell, common, lru_cache from shadowsocks.common import pre_parse_header, parse_header @@ -93,6 +94,30 @@ BUF_SIZE = 32 * 1024 UDP_MAX_BUF_SIZE = 65536 +class SpeedTester(object): + def __init__(self, max_speed = 0): + self.max_speed = max_speed + self.timeout = 3 + self._cache = deque() + self.sum_len = 0 + + def add(self, data_len): + if self.max_speed > 0: + self._cache.append((time.time(), data_len)) + self.sum_len += data_len + + def isExceed(self): + if self.max_speed > 0: + if self.sum_len > 0: + if self._cache[0][0] + self.timeout < time.time(): + self.sum_len -= self._cache[0][1] + self._cache.popleft() + if self.sum_len > 0: + t = max(time.time() - self._cache[0][0], 0.1) + speed = (self.sum_len - self._cache[0][1]) / (time.time() - self._cache[0][0]) + return speed >= self.max_speed + return False + class TCPRelayHandler(object): def __init__(self, server, fd_to_handlers, loop, local_sock, config, dns_resolver, is_local): @@ -189,6 +214,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._update_activity() self._server.add_connection(1) self._server.stat_add(self._client_address[0], 1) + self.speed_tester_u = SpeedTester(config.get("speed_limit_per_con", 0)) + self.speed_tester_d = SpeedTester(config.get("speed_limit_per_con", 0)) def __hash__(self): # default __hash__ is id / 16 @@ -725,6 +752,8 @@ def _on_local_read(self): if not data: self.destroy() return + + self.speed_tester_u.add(len(data)) ogn_data = data if not is_local: if self._encryptor is not None: @@ -819,6 +848,8 @@ def _on_remote_read(self, is_remote_sock): if not data: self.destroy() return + + self.speed_tester_d.add(len(data)) if self._encryptor is not None: if self._is_local: try: @@ -900,38 +931,49 @@ def _on_remote_error(self): def handle_event(self, sock, event): # handle all events in this handler and dispatch them to methods + handle = False if self._stage == STAGE_DESTROYED: logging.debug('ignore handle_event: destroyed') - return + return True if self._user is not None and self._user not in self._server.server_users: self.destroy() - return + return True # order is important if sock == self._remote_sock or sock == self._remote_sock_v6: if event & eventloop.POLL_ERR: + handle = True self._on_remote_error() if self._stage == STAGE_DESTROYED: - return + return True if event & (eventloop.POLL_IN | eventloop.POLL_HUP): - self._on_remote_read(sock == self._remote_sock) - if self._stage == STAGE_DESTROYED: - return + if not self.speed_tester_d.isExceed(): + handle = True + self._on_remote_read(sock == self._remote_sock) + if self._stage == STAGE_DESTROYED: + return True if event & eventloop.POLL_OUT: + handle = True self._on_remote_write() elif sock == self._local_sock: if event & eventloop.POLL_ERR: + handle = True self._on_local_error() if self._stage == STAGE_DESTROYED: - return + return True if event & (eventloop.POLL_IN | eventloop.POLL_HUP): - self._on_local_read() - if self._stage == STAGE_DESTROYED: - return + if not self.speed_tester_u.isExceed(): + handle = True + self._on_local_read() + if self._stage == STAGE_DESTROYED: + return True if event & eventloop.POLL_OUT: + handle = True self._on_local_write() else: logging.warn('unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) + return handle + def _log_error(self, e): logging.error('%s when handling connection from %s:%d' % (e, self._client_address[0], self._client_address[1])) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 9ee5ccf8..3d71b872 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -782,35 +782,44 @@ def handle_client(self, addr, cmd, request_id, data): def handle_event(self, sock, event): # handle all events in this handler and dispatch them to methods + handle = False if self._stage == STAGE_DESTROYED: logging.debug('ignore handle_event: destroyed') - return + return True # order is important if sock == self._remote_sock: if event & eventloop.POLL_ERR: + handle = True self._on_remote_error() if self._stage == STAGE_DESTROYED: - return + return True if event & (eventloop.POLL_IN | eventloop.POLL_HUP): + handle = True self._on_remote_read() if self._stage == STAGE_DESTROYED: - return + return True if event & eventloop.POLL_OUT: + handle = True self._on_remote_write() elif sock == self._local_sock: if event & eventloop.POLL_ERR: + handle = True self._on_local_error() if self._stage == STAGE_DESTROYED: - return + return True if event & (eventloop.POLL_IN | eventloop.POLL_HUP): + handle = True self._on_local_read() if self._stage == STAGE_DESTROYED: - return + return True if event & eventloop.POLL_OUT: + handle = True self._on_local_write() else: logging.warn('unknown socket') + return handle + def _log_error(self, e): logging.error('%s when handling connection from %s' % (e, self._client_address.keys())) From 85955d6e6dbd142d2ff7dd5b08458387d6308339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 12 Feb 2017 22:46:09 +0800 Subject: [PATCH 369/487] speed limit in KiB --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 431fe9cc..f6e9f7ad 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -96,7 +96,7 @@ class SpeedTester(object): def __init__(self, max_speed = 0): - self.max_speed = max_speed + self.max_speed = max_speed * 1024 self.timeout = 3 self._cache = deque() self.sum_len = 0 From 27232951015c68e04b47560afe9abf0242912963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 13 Feb 2017 01:38:48 +0800 Subject: [PATCH 370/487] fix CPU usage --- shadowsocks/eventloop.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index a9560fa5..5707154d 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -222,8 +222,8 @@ def run(self): for callback in self._periodic_callbacks: callback() self._last_time = now - if events and (handle is False): - time.sleep(0.01) + if events and not handle: + time.sleep(0.02) def __del__(self): self._impl.close() From 6732244ddf3fb98622efc5dee5a3e7d0d16fae3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 13 Feb 2017 02:14:21 +0800 Subject: [PATCH 371/487] lower latency --- shadowsocks/eventloop.py | 2 +- shadowsocks/tcprelay.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 5707154d..e36f8584 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -223,7 +223,7 @@ def run(self): callback() self._last_time = now if events and not handle: - time.sleep(0.02) + time.sleep(0.001) def __del__(self): self._impl.close() diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f6e9f7ad..7f2d3c9a 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -97,7 +97,7 @@ class SpeedTester(object): def __init__(self, max_speed = 0): self.max_speed = max_speed * 1024 - self.timeout = 3 + self.timeout = 5 self._cache = deque() self.sum_len = 0 @@ -109,12 +109,12 @@ def add(self, data_len): def isExceed(self): if self.max_speed > 0: if self.sum_len > 0: - if self._cache[0][0] + self.timeout < time.time(): + cut_t = time.time() + t = max(cut_t - self._cache[0][0], 0.01) + speed = self.sum_len / t + if self._cache[0][0] + self.timeout < cut_t: self.sum_len -= self._cache[0][1] self._cache.popleft() - if self.sum_len > 0: - t = max(time.time() - self._cache[0][0], 0.1) - speed = (self.sum_len - self._cache[0][1]) / (time.time() - self._cache[0][0]) return speed >= self.max_speed return False From 74083af47212ccbe995e2da0f4fc205fdb71ca49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 14 Feb 2017 02:45:40 +0800 Subject: [PATCH 372/487] process DNS --- shadowsocks/udprelay.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 3d71b872..c337eb6d 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -902,7 +902,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self._cache_dns_client = lru_cache.LRUCache(timeout=10, close_callback=self._close_client_pair) self._client_fd_to_server_addr = {} - self._dns_cache = lru_cache.LRUCache(timeout=300) + self._dns_cache = lru_cache.LRUCache(timeout=1800) self._eventloop = None self._closed = False self.server_transfer_ul = 0 @@ -1158,20 +1158,28 @@ def _handle_server(self): connecttype, dest_addr, dest_port, header_length = header_result if self._is_local: + connecttype = 3 server_addr, server_port = self._get_a_server() else: server_addr, server_port = dest_addr, dest_port - addrs = self._dns_cache.get(server_addr, None) - if addrs is None: - # TODO async getaddrinfo + if (connecttype & 7) == 3: + addrs = self._dns_cache.get(server_addr, None) + if addrs is None: + # TODO async getaddrinfo + addrs = socket.getaddrinfo(server_addr, server_port, 0, + socket.SOCK_DGRAM, socket.SOL_UDP) + if not addrs: + # drop + return + else: + self._dns_cache[server_addr] = addrs + else: addrs = socket.getaddrinfo(server_addr, server_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) if not addrs: # drop return - else: - self._dns_cache[server_addr] = addrs af, socktype, proto, canonname, sa = addrs[0] key = client_key(r_addr, af) From c4c1cd6dc1d14dfe016f837fdf74ac38f892a3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 14 Feb 2017 20:23:45 +0800 Subject: [PATCH 373/487] refine auth_aes128_* & http_* --- shadowsocks/obfsplugin/auth.py | 12 ++++++------ shadowsocks/obfsplugin/http_simple.py | 9 +++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index b787b33a..943caa1e 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1319,12 +1319,12 @@ def server_post_decrypt(self, buf): sendback = False if not self.has_recv_header: - if len(self.recv_buf) < 7: - return (b'', False) - mac_key = self.server_info.recv_iv + self.server_info.key - sha1data = hmac.new(mac_key, self.recv_buf[:1], self.hashfunc).digest()[:6] - if sha1data != self.recv_buf[1:7]: - return self.not_match_return(self.recv_buf) + if len(self.recv_buf) >= 7 or len(self.recv_buf) in [2, 3]: + recv_len = min(len(self.recv_buf), 7) + mac_key = self.server_info.recv_iv + self.server_info.key + sha1data = hmac.new(mac_key, self.recv_buf[:1], self.hashfunc).digest()[:recv_len - 1] + if sha1data != self.recv_buf[1:recv_len]: + return self.not_match_return(self.recv_buf) if len(self.recv_buf) < 31: return (b'', False) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 6549f779..19e2a759 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -168,6 +168,11 @@ def not_match_return(self, buf): return (b'E'*2048, False, False) return (buf, True, False) + def error_return(self, buf): + self.has_sent_header = True + self.has_recv_header = True + return (b'E'*2048, False, False) + def server_decode(self, buf): if self.has_recv_header: return (buf, True, False) @@ -199,10 +204,10 @@ def server_decode(self, buf): if host not in hosts: return self.not_match_return(buf) if len(ret_buf) < 4: - return self.not_match_return(buf) + return self.error_return(buf) if len(datas) > 1: ret_buf += datas[1] - if len(ret_buf) >= 7: + if len(ret_buf) >= 13: self.has_recv_header = True return (ret_buf, True, False) return self.not_match_return(buf) From f296751def517160174975a604a7f5faf2f4382b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 14 Feb 2017 21:36:38 +0800 Subject: [PATCH 374/487] bandwidth limit of the user --- config.json | 1 + shadowsocks/eventloop.py | 2 +- shadowsocks/tcprelay.py | 42 ++++++++++++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/config.json b/config.json index 0d0d13ea..ef4c79eb 100644 --- a/config.json +++ b/config.json @@ -14,6 +14,7 @@ "obfs": "tls1.2_ticket_auth_compatible", "obfs_param": "", "speed_limit_per_con": 0, + "speed_limit_per_user": 0, "dns_ipv6": false, "connect_verbose_info": 0, diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index e36f8584..6e2f22f2 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -53,7 +53,7 @@ } # we check timeouts every TIMEOUT_PRECISION seconds -TIMEOUT_PRECISION = 5 +TIMEOUT_PRECISION = 2 class KqueueLoop(object): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7f2d3c9a..fcaadb07 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -133,6 +133,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._client_address = local_sock.getpeername()[:2] self._accept_address = local_sock.getsockname()[:2] self._user = None + self._user_id = server._listen_port # TCP Relay works as either sslocal or ssserver # if is_local, this is sslocal @@ -238,6 +239,7 @@ def _get_a_server(self): def _update_user(self, user): self._user = user + self._user_id = struct.unpack(' Date: Tue, 14 Feb 2017 23:28:49 +0800 Subject: [PATCH 375/487] timeout set to 1 --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index fcaadb07..f4551f24 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -97,7 +97,7 @@ class SpeedTester(object): def __init__(self, max_speed = 0): self.max_speed = max_speed * 1024 - self.timeout = 5 + self.timeout = 1 self._cache = deque() self.sum_len = 0 From e1b199fc44008f226eb786f9aaa3b210aa3ed8b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 19 Feb 2017 10:27:25 +0800 Subject: [PATCH 376/487] add additional_ports in config using user's password as protocol param in multi-user mode --- config.json | 5 +++-- db_transfer.py | 32 ++++++++++++++++++++++++-------- mujson_mgr.py | 38 +++++++++++++++++++++----------------- server_pool.py | 10 +++++----- shadowsocks/shell.py | 3 ++- shadowsocks/tcprelay.py | 30 +++++++++++++++++++++--------- shadowsocks/udprelay.py | 21 ++++++++++++--------- 7 files changed, 88 insertions(+), 51 deletions(-) diff --git a/config.json b/config.json index ef4c79eb..988100f7 100644 --- a/config.json +++ b/config.json @@ -6,8 +6,6 @@ "local_port": 1080, "password": "m", - "timeout": 120, - "udp_timeout": 60, "method": "aes-128-ctr", "protocol": "auth_aes128_md5", "protocol_param": "", @@ -16,6 +14,9 @@ "speed_limit_per_con": 0, "speed_limit_per_user": 0, + "additional_ports" : {}, // only works under multi-user mode + "timeout": 120, + "udp_timeout": 60, "dns_ipv6": false, "connect_verbose_info": 0, "redirect": "", diff --git a/db_transfer.py b/db_transfer.py index 4d6088f9..5862de79 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -24,6 +24,7 @@ def __init__(self): self.port_uid_table = {} #端口到uid的映射（仅v3以上有用） self.onlineuser_cache = lru_cache.LRUCache(timeout=60*30) #用户在线状态记录 self.pull_ok = False #记录是否已经拉出过数据 + self.mu_ports = {} def load_cfg(self): pass @@ -41,7 +42,7 @@ def push_db_all_user(self): dt_transfer[id] = [self.last_get_transfer[id][0] - last_transfer[id][0], self.last_get_transfer[id][1] - last_transfer[id][1]] for id in curr_transfer.keys(): - if id in self.force_update_transfer: + if id in self.force_update_transfer or id in self.mu_ports: continue #算出与上次记录的流量差值，保存于dt_transfer表 if id in last_transfer: @@ -95,11 +96,13 @@ def del_server_out_of_bound_safe(self, last_rows, rows): port = row['port'] passwd = common.to_bytes(row['passwd']) + if hasattr(passwd, 'encode'): + passwd = passwd.encode('utf-8') cfg = {'password': passwd} if 'id' in row: self.port_uid_table[row['port']] = row['id'] - read_config_keys = ['method', 'obfs', 'obfs_param', 'protocol', 'protocol_param', 'forbidden_ip', 'forbidden_port'] + read_config_keys = ['method', 'obfs', 'obfs_param', 'protocol', 'protocol_param', 'forbidden_ip', 'forbidden_port', 'speed_limit_per_con', 'speed_limit_per_user'] for name in read_config_keys: if name in row and row[name]: cfg[name] = row[name] @@ -116,10 +119,11 @@ def del_server_out_of_bound_safe(self, last_rows, rows): continue if allow: - allow_users[port] = 1 + allow_users[port] = passwd if 'protocol' in cfg and 'protocol_param' in cfg and common.to_str(cfg['protocol']) in ['auth_aes128_md5', 'auth_aes128_sha1']: if '#' in common.to_str(cfg['protocol_param']): - mu_servers[port] = 1 + mu_servers[port] = passwd + del allow_users[port] cfgchange = False if port in ServerPool.get_instance().tcp_servers_pool: @@ -177,10 +181,11 @@ def del_server_out_of_bound_safe(self, last_rows, rows): passwd, cfg = new_servers[port] self.new_server(port, passwd, cfg) - if isinstance(self, MuJsonTransfer): # works in MuJsonTransfer only - logging.debug('db allow users %s \nmu_servers %s' % (allow_users, mu_servers)) - for port in mu_servers: - ServerPool.get_instance().update_mu_server(port, None, allow_users) + logging.debug('db allow users %s \nmu_servers %s' % (allow_users, mu_servers)) + for port in mu_servers: + ServerPool.get_instance().update_mu_users(port, allow_users) + + self.mu_ports = mu_servers def clear_cache(self, port): if port in self.force_update_transfer: del self.force_update_transfer[port] @@ -237,6 +242,17 @@ def thread_db(obj): rows = db_instance.pull_db_all_user() if rows: db_instance.pull_ok = True + config = shell.get_config(False) + for port in config['additional_ports']: + val = config['additional_ports'][port] + val['port'] = int(port) + val['enable'] = 1 + val['transfer_enable'] = 1024 ** 7 + val['u'] = 0 + val['d'] = 0 + if "password" in val: + val["passwd"] = val["password"] + rows.append(val) db_instance.del_server_out_of_bound_safe(last_rows, rows) last_rows = rows except Exception as e: diff --git a/mujson_mgr.py b/mujson_mgr.py index 5d85798c..760763ce 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -68,24 +68,17 @@ def ssrlink(self, user, encode, muid): obfs = user.get('obfs', '') protocol = protocol.replace("_compatible", "") obfs = obfs.replace("_compatible", "") - link = "%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace("=", "")) + protocol_param = '' if muid is not None: - protocol_param = user.get('protocol_param', '') - param = protocol_param.split('#') + protocol_param_ = user.get('protocol_param', '') + param = protocol_param_.split('#') if len(param) == 2: - user_dict = {} - user_list = param[1].split(',') - if user_list: - for userinfo in user_list: - items = userinfo.split(':') - if len(items) == 2: - user_int_id = int(items[0]) - passwd = items[1] - user_dict[user_int_id] = passwd - if muid in user_dict: - param = str(muid) + ':' + user_dict[muid] + for row in self.data.json: + if int(row['port']) == muid: + param = str(muid) + ':' + row['passwd'] protocol_param = '/?protoparam=' + base64.urlsafe_b64encode(common.to_bytes(param)).replace("=", "") - link += protocol_param + break + link = ("%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace("=", ""))) + protocol_param return "ssr://" + (encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) def userinfo(self, user, muid = None): @@ -98,8 +91,19 @@ def userinfo(self, user, muid = None): if key in ['enable'] or key not in user: continue ret += '\n' - if key in ['transfer_enable', 'u', 'd']: - val = user[key] + if (muid is not None) and (key in ['protocol_param']): + for row in self.data.json: + if int(row['port']) == muid: + ret += " %s : %s" % (key, str(muid) + ':' + row['passwd']) + break + elif key in ['transfer_enable', 'u', 'd']: + if muid is not None: + for row in self.data.json: + if int(row['port']) == muid: + val = row[key] + break + else: + val = user[key] if val / 1024 < 4: ret += " %s : %s" % (key, val) elif val / 1024 ** 2 < 4: diff --git a/server_pool.py b/server_pool.py index 45fe6a13..2e6a85bc 100644 --- a/server_pool.py +++ b/server_pool.py @@ -214,24 +214,24 @@ def cb_del_server(self, port): return True - def update_mu_server(self, port, protocol_param, acl): + def update_mu_users(self, port, users): port = int(port) if port in self.tcp_servers_pool: try: - self.tcp_servers_pool[port].update_users(protocol_param, acl) + self.tcp_servers_pool[port].update_users(users) except Exception as e: logging.warn(e) try: - self.udp_servers_pool[port].update_users(protocol_param, acl) + self.udp_servers_pool[port].update_users(users) except Exception as e: logging.warn(e) if port in self.tcp_ipv6_servers_pool: try: - self.tcp_ipv6_servers_pool[port].update_users(protocol_param, acl) + self.tcp_ipv6_servers_pool[port].update_users(users) except Exception as e: logging.warn(e) try: - self.udp_ipv6_servers_pool[port].update_users(protocol_param, acl) + self.udp_ipv6_servers_pool[port].update_users(users) except Exception as e: logging.warn(e) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 2fb268a6..41d2323f 100755 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -168,7 +168,7 @@ def get_config(is_local): if config_path: - logging.info('loading config from %s' % config_path) + logging.debug('loading config from %s' % config_path) with open(config_path, 'rb') as f: try: config = parse_json_in_str(remove_comment(f.read().decode('utf8'))) @@ -244,6 +244,7 @@ def get_config(is_local): config['obfs'] = to_str(config.get('obfs', 'plain')) config['obfs_param'] = to_str(config.get('obfs_param', '')) config['port_password'] = config.get('port_password', None) + config['additional_ports'] = config.get('additional_ports', {}) config['timeout'] = int(config.get('timeout', 300)) config['udp_timeout'] = int(config.get('udp_timeout', 120)) config['udp_cache'] = int(config.get('udp_cache', 64)) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f4551f24..5afefc4b 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -101,6 +101,9 @@ def __init__(self, max_speed = 0): self._cache = deque() self.sum_len = 0 + def update_limit(self, max_speed): + self.max_speed = max_speed * 1024 + def add(self, data_len): if self.max_speed > 0: self._cache.append((time.time(), data_len)) @@ -1057,8 +1060,6 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self.mu = False self._speed_tester_u = {} self._speed_tester_d = {} - self.update_users_protocol_param = None - self.update_users_acl = None self.server_connections = 0 self.protocol_data = obfs.obfs(config['protocol']).init_data() self.obfs_data = obfs.obfs(config['obfs']).init_data() @@ -1145,9 +1146,18 @@ def _update_users(self, protocol_param, acl): passwd = items[1] self.add_user(uid, passwd) - def update_users(self, protocol_param, acl): - self.update_users_protocol_param = protocol_param - self.update_users_acl = acl + def update_user(self, id, passwd): + uid = struct.pack(' Date: Thu, 2 Mar 2017 23:20:06 +0800 Subject: [PATCH 377/487] fix str type under py3 --- mujson_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 760763ce..d7f44214 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -76,7 +76,7 @@ def ssrlink(self, user, encode, muid): for row in self.data.json: if int(row['port']) == muid: param = str(muid) + ':' + row['passwd'] - protocol_param = '/?protoparam=' + base64.urlsafe_b64encode(common.to_bytes(param)).replace("=", "") + protocol_param = '/?protoparam=' + common.to_str(base64.urlsafe_b64encode(common.to_bytes(param))).replace("=", "") break link = ("%s:%s:%s:%s:%s:%s" % (self.server_addr, user['port'], protocol, user['method'], obfs, common.to_str(base64.urlsafe_b64encode(common.to_bytes(user['passwd']))).replace("=", ""))) + protocol_param return "ssr://" + (encode and common.to_str(base64.urlsafe_b64encode(common.to_bytes(link))).replace("=", "") or link) From 1355c273db2957e572a9e31cfff4bf112aa403c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 16 Mar 2017 00:35:19 +0800 Subject: [PATCH 378/487] remove bom --- mujson_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index d7f44214..320d1277 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python # -*- coding: UTF-8 -*- import traceback From e3d6efbc7a77337e0e545b99275a52b7fd2fdcd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 16 Mar 2017 00:39:49 +0800 Subject: [PATCH 379/487] bump --- CHANGES | 5 +++++ shadowsocks/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index db0ae4bb..8c7bfc59 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +3.1.0 2017-03-16 +- add "glzjinmod" interface +- rate limit +- add additional_ports in config + 3.0.4 2017-01-08 - multi-user in single port diff --git a/shadowsocks/version.py b/shadowsocks/version.py index db6c4a10..5ff56daa 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.0.2 2017-01-08' + return '3.1.0 2017-03-16' From e19f9e70e747323d4b870766afbe0bc24455e1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 18 Mar 2017 17:58:33 +0800 Subject: [PATCH 380/487] add "New session ticket" --- shadowsocks/obfsplugin/obfs_tls.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 18839106..ffb7646c 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -152,6 +152,8 @@ def client_decode(self, buf): verify = buf[11:33] if hmac.new(self.server_info.key + self.server_info.data.client_id, verify, hashlib.sha1).digest()[:10] != buf[33:43]: raise Exception('client_decode data error') + if hmac.new(self.server_info.key + self.server_info.data.client_id, buf[:-10], hashlib.sha1).digest()[:10] != buf[-10:]: + raise Exception('client_decode data error') return (b'', True) def server_encode(self, buf): @@ -170,6 +172,10 @@ def server_encode(self, buf): data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"c02f000005ff01000100") data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data + if random.randint(0, 255) < 128: + ticket = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 256) + 64) + ticket = struct.pack('>H', len(ticket) + 4) b"\x04\x00" + struct.pack('>H', len(ticket)) + data += b"\x16" + self.tls_version + ticket #New session ticket data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] From 6f35105ea28d22e8b2f8f08fda29d05748aeb3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 18 Mar 2017 18:00:36 +0800 Subject: [PATCH 381/487] fix syntax --- shadowsocks/obfsplugin/obfs_tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index ffb7646c..b76b08b4 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -174,7 +174,7 @@ def server_encode(self, buf): data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data if random.randint(0, 255) < 128: ticket = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 256) + 64) - ticket = struct.pack('>H', len(ticket) + 4) b"\x04\x00" + struct.pack('>H', len(ticket)) + ticket = struct.pack('>H', len(ticket) + 4) + b"\x04\x00" + struct.pack('>H', len(ticket)) data += b"\x16" + self.tls_version + ticket #New session ticket data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished From caaacc7090e493cb6b2c11db9c9af6e45840b5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 18 Mar 2017 21:35:39 +0800 Subject: [PATCH 382/487] accept any size of Finish message --- shadowsocks/obfsplugin/obfs_tls.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index b76b08b4..a796f1f6 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -172,12 +172,13 @@ def server_encode(self, buf): data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"c02f000005ff01000100") data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data - if random.randint(0, 255) < 128: - ticket = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 256) + 64) + if random.randint(0, 8) < 1: + ticket = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 164) * 2 + 64) ticket = struct.pack('>H', len(ticket) + 4) + b"\x04\x00" + struct.pack('>H', len(ticket)) data += b"\x16" + self.tls_version + ticket #New session ticket data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec - data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished + finish_len = random.choice([32, 40]) + data += b"\x16" + self.tls_version + struct.pack('>H', finish_len) + os.urandom(finish_len - 10) #Finished data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] return data @@ -215,7 +216,7 @@ def server_decode(self, buf): if not match_begin(buf, b"\x14" + self.tls_version + b"\x00\x01\x01"): #ChangeCipherSpec raise Exception('server_decode data error') buf = buf[6:] - if not match_begin(buf, b"\x16" + self.tls_version + b"\x00\x20"): #Finished + if not match_begin(buf, b"\x16" + self.tls_version + b"\x00"): #Finished raise Exception('server_decode data error') if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: raise Exception('server_decode data error') From 6712dd62c0cc515ecd0f2b99d9576bcab0019806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 18 Mar 2017 21:49:50 +0800 Subject: [PATCH 383/487] fix accept any size of Finish message --- shadowsocks/obfsplugin/obfs_tls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index a796f1f6..7d35f7cc 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -210,7 +210,6 @@ def server_decode(self, buf): self.recv_buffer += buf buf = self.recv_buffer verify = buf - verify_len = 43 - 10 if len(buf) < 43: raise Exception('server_decode data error') if not match_begin(buf, b"\x14" + self.tls_version + b"\x00\x01\x01"): #ChangeCipherSpec @@ -218,11 +217,12 @@ def server_decode(self, buf): buf = buf[6:] if not match_begin(buf, b"\x16" + self.tls_version + b"\x00"): #Finished raise Exception('server_decode data error') - if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: + verify_len = struct.unpack('>H', buf[3:5])[0] + 1 # 11 - 10 + if len(verify) < verify_len + 10: raise Exception('server_decode data error') - if len(buf) < 37: + if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: raise Exception('server_decode data error') - self.recv_buffer = buf[37:] + self.recv_buffer = verify[verify_len + 10:] self.handshake_status = 8 return self.server_decode(b'') From 1c58b10dd09788d7039993ae921e398afda9da16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 19 Mar 2017 18:28:43 +0800 Subject: [PATCH 384/487] ignore bind 10.0.0.0/8 and 192.168.0.0/16 --- shadowsocks/shell.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 41d2323f..1540412a 100755 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -250,8 +250,8 @@ def get_config(is_local): config['udp_cache'] = int(config.get('udp_cache', 64)) config['fast_open'] = config.get('fast_open', False) config['workers'] = config.get('workers', 1) - config['pid-file'] = config.get('pid-file', '/var/run/shadowsocks.pid') - config['log-file'] = config.get('log-file', '/var/log/shadowsocks.log') + config['pid-file'] = config.get('pid-file', '/var/run/shadowsocksr.pid') + config['log-file'] = config.get('log-file', '/var/log/shadowsocksr.log') config['verbose'] = config.get('verbose', False) config['connect_verbose_info'] = config.get('connect_verbose_info', 0) config['local_address'] = to_str(config.get('local_address', '127.0.0.1')) @@ -278,7 +278,7 @@ def get_config(is_local): sys.exit(2) try: config['ignore_bind'] = \ - IPNetwork(config.get('ignore_bind', '127.0.0.0/8,::1/128')) + IPNetwork(config.get('ignore_bind', '127.0.0.0/8,::1/128,10.0.0.0/8,192.168.0.0/16')) except Exception as e: logging.error(e) sys.exit(2) From 63a6de5d26f5d44595187e551957b0f0957df65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 23 Mar 2017 13:16:58 +0800 Subject: [PATCH 385/487] fix host type in http_simple --- shadowsocks/obfsplugin/http_simple.py | 2 +- shadowsocks/server.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 19e2a759..f84beea1 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -159,7 +159,7 @@ def get_host_from_http_header(self, buf): if lines and len(lines) > 1: for line in lines: if match_begin(line, b"Host: "): - return line[6:] + return common.to_str(line[6:]) def not_match_return(self, buf): self.has_sent_header = True diff --git a/shadowsocks/server.py b/shadowsocks/server.py index d3ce95cb..f4dfdd5f 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -96,8 +96,8 @@ def main(): protocol_param = password_obfs.get('protocol_param', protocol_param) obfs = password_obfs.get('obfs', obfs) obfs_param = password_obfs.get('obfs_param', obfs_param) - bind = password_obfs.get('bind', bind) - bindv6 = password_obfs.get('bindv6', bindv6) + bind = password_obfs.get('out_bind', bind) + bindv6 = password_obfs.get('out_bindv6', bindv6) else: password = password_obfs a_config = config.copy() From bf0055be40184670b26d86bb3c257fc803508fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 23 Mar 2017 17:19:15 +0800 Subject: [PATCH 386/487] fix Dockerfile --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a065b7a9..45045981 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,9 @@ FROM alpine ENV SERVER_ADDR 0.0.0.0 ENV SERVER_PORT 51348 ENV PASSWORD= -ENV METHOD aes-256-cfb -ENV PROTOCOL origin -ENV OBFS http_simple +ENV METHOD aes-128-ctr +ENV PROTOCOL auth_aes128_md5 +ENV OBFS tls1.2_ticket_auth_compatible ENV TIMEOUT 300 ENV DNS_ADDR 8.8.8.8 ENV DNS_ADDR_2 8.8.4.4 @@ -22,11 +22,11 @@ RUN apk update \ RUN wget --no-check-certificate https://github.com/breakwa11/shadowsocks/archive/manyuser.zip -O /tmp/manyuser.zip \ && unzip -d /tmp /tmp/manyuser.zip \ - && mv /tmp/shadowsocks-manyuser/shadowsocks ~/shadowsocks \ + && mv /tmp/shadowsocksr-manyuser/shadowsocks ~/shadowsocksr \ && rm -rf /tmp/* -WORKDIR ~/shadowsocks +WORKDIR ~/shadowsocksr -CMD python ~/shadowsocks/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS +CMD python ~/shadowsocksr/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS From 6cbcd855ff3ea12435bd4837215fcce3c4282ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 24 Mar 2017 17:07:52 +0800 Subject: [PATCH 387/487] better rand size --- shadowsocks/obfsplugin/auth.py | 31 +++++++++++++++++++------------ shadowsocks/tcprelay.py | 4 ++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 943caa1e..70f4023d 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1172,6 +1172,7 @@ def __init__(self, method, hashfunc): self.recv_id = 1 self.user_id = None self.user_key = None + self.last_rnd_len = 0 def init_data(self): return obfs_auth_mu_data() @@ -1184,21 +1185,25 @@ def set_server_info(self, server_info): max_client = 64 self.server_info.data.set_max_client(max_client) + def rnd_data_len(self, buf_size): + if buf_size > 1300 or self.last_rnd_len > 1300: + return 0 + if buf_size > 1100: + return common.ord(os.urandom(1)[0]) % 128 + #self.pack_id + if buf_size > 400: + return struct.unpack('>H', os.urandom(2))[0] % 256 + else: + return struct.unpack('>H', os.urandom(2))[0] % 1024 + def rnd_data(self, buf_size): - if buf_size > 1200: - return b'\x01' + data_len = self.rnd_data_len(buf_size) + self.last_rnd_len = data_len - if self.pack_id > 4: - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 32) - elif buf_size > 900: - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) - else: - rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) + if data_len < 128: + return common.chr(data_len + 1) + os.urandom(data_len) - if len(rnd_data) < 128: - return common.chr(len(rnd_data) + 1) + rnd_data - else: - return common.chr(255) + struct.pack(' self.unit_len: ret += self.pack_data(buf[:self.unit_len]) buf = buf[self.unit_len:] diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 5afefc4b..7f0523af 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -165,7 +165,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.key_str = common.to_bytes(config['password']) server_info.key = self._encryptor.cipher_key server_info.head_len = 30 - server_info.tcp_mss = 1460 + server_info.tcp_mss = 1448 self._obfs.set_server_info(server_info) self._protocol = obfs.obfs(config['protocol']) @@ -183,7 +183,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.key_str = common.to_bytes(config['password']) server_info.key = self._encryptor.cipher_key server_info.head_len = 30 - server_info.tcp_mss = 1460 + server_info.tcp_mss = 1448 self._protocol.set_server_info(server_info) self._redir_list = config.get('redirect', ["*#0.0.0.0:0"]) From 1b1c1fbac6eb83ec9edf4294e8fcca6ddc464adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 25 Mar 2017 01:53:11 +0800 Subject: [PATCH 388/487] improve rand size --- shadowsocks/obfsplugin/auth.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 70f4023d..83be8249 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1185,8 +1185,8 @@ def set_server_info(self, server_info): max_client = 64 self.server_info.data.set_max_client(max_client) - def rnd_data_len(self, buf_size): - if buf_size > 1300 or self.last_rnd_len > 1300: + def rnd_data_len(self, buf_size, full_buf_size): + if buf_size > 1300 or self.last_rnd_len > 1300 or full_buf_size > 1492: return 0 if buf_size > 1100: return common.ord(os.urandom(1)[0]) % 128 @@ -1196,17 +1196,16 @@ def rnd_data_len(self, buf_size): else: return struct.unpack('>H', os.urandom(2))[0] % 1024 - def rnd_data(self, buf_size): - data_len = self.rnd_data_len(buf_size) - self.last_rnd_len = data_len + def rnd_data(self, buf_size, full_buf_size): + data_len = self.rnd_data_len(buf_size, full_buf_size) if data_len < 128: return common.chr(data_len + 1) + os.urandom(data_len) return common.chr(255) + struct.pack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) + ret += self.pack_data(buf[:self.unit_len], ogn_data_len) buf = buf[self.unit_len:] - ret += self.pack_data(buf) + ret += self.pack_data(buf, ogn_data_len) + self.last_rnd_len = ogn_data_len return ret def client_post_decrypt(self, buf): @@ -1311,11 +1311,12 @@ def server_pre_encrypt(self, buf): if self.raw_trans: return buf ret = b'' - self.last_rnd_len = 0 + ogn_data_len = len(buf) while len(buf) > self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) + ret += self.pack_data(buf[:self.unit_len], ogn_data_len) buf = buf[self.unit_len:] - ret += self.pack_data(buf) + ret += self.pack_data(buf, ogn_data_len) + self.last_rnd_len = ogn_data_len return ret def server_post_decrypt(self, buf): From 63cceb29f9ab2b26dde4c52c4916e473e4428b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 25 Mar 2017 01:54:23 +0800 Subject: [PATCH 389/487] fix Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 45045981..7e7383f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,11 +22,11 @@ RUN apk update \ RUN wget --no-check-certificate https://github.com/breakwa11/shadowsocks/archive/manyuser.zip -O /tmp/manyuser.zip \ && unzip -d /tmp /tmp/manyuser.zip \ - && mv /tmp/shadowsocksr-manyuser/shadowsocks ~/shadowsocksr \ + && mv /tmp/shadowsocksr-manyuser/shadowsocks ~/shadowsocks \ && rm -rf /tmp/* -WORKDIR ~/shadowsocksr +WORKDIR ~/shadowsocks -CMD python ~/shadowsocksr/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS +CMD python ~/shadowsocks/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS From 76fd4296a588abcb5501c2f9cb3836cae538f7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 25 Mar 2017 02:17:59 +0800 Subject: [PATCH 390/487] bump improve rand size --- CHANGES | 6 ++++++ shadowsocks/obfsplugin/auth.py | 2 +- shadowsocks/tcprelay.py | 1 + shadowsocks/udprelay.py | 1 + shadowsocks/version.py | 2 +- 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 8c7bfc59..05bda56b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +3.1.1 2017-03-25 +- add "New session ticket" +- ignore bind 10.0.0.0/8 and 192.168.0.0/16 by default +- improve rand size under auth_aes128_* +- fix bugs + 3.1.0 2017-03-16 - add "glzjinmod" interface - rate limit diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 83be8249..ad2fbf8a 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1186,7 +1186,7 @@ def set_server_info(self, server_info): self.server_info.data.set_max_client(max_client) def rnd_data_len(self, buf_size, full_buf_size): - if buf_size > 1300 or self.last_rnd_len > 1300 or full_buf_size > 1492: + if buf_size > 1300 or self.last_rnd_len > 1300 or full_buf_size >= self.server_info.buffer_size: return 0 if buf_size > 1100: return common.ord(os.urandom(1)[0]) % 128 diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7f0523af..7f01f712 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -184,6 +184,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.key = self._encryptor.cipher_key server_info.head_len = 30 server_info.tcp_mss = 1448 + server_info.buffer_size = BUF_SIZE self._protocol.set_server_info(server_info) self._redir_list = config.get('redirect', ["*#0.0.0.0:0"]) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 3dd77609..3d8df3b3 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -928,6 +928,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun server_info.key = encrypt.encrypt_key(self._password, self._method) server_info.head_len = 30 server_info.tcp_mss = 1440 + server_info.buffer_size = BUF_SIZE self._protocol.set_server_info(server_info) self._sockets = set() diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 5ff56daa..eed0e4ec 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.1.0 2017-03-16' + return '3.1.1 2017-03-25' From 922cf462d683b78b55a7288e8af2dc535049e6f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 26 Mar 2017 00:00:01 +0800 Subject: [PATCH 391/487] improve rand --- shadowsocks/obfsplugin/auth.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index ad2fbf8a..98a958dc 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -26,6 +26,7 @@ import time import datetime import random +import math import struct import zlib import hmac @@ -1185,16 +1186,24 @@ def set_server_info(self, server_info): max_client = 64 self.server_info.data.set_max_client(max_client) + def trapezoid_random_float(self, d): + if d == 0: + return random.random() + s = random.random() + a = 1 - d + return (math.sqrt(a * a + 4 * d * s) - a) / (2 * d) + + def trapezoid_random_int(self, max_val, d): + v = self.trapezoid_random_float(d) + return int(v * max_val) + def rnd_data_len(self, buf_size, full_buf_size): - if buf_size > 1300 or self.last_rnd_len > 1300 or full_buf_size >= self.server_info.buffer_size: + rev_len = self.server_info.tcp_mss - buf_size - 9 + if rev_len <= 0 or self.last_rnd_len >= self.server_info.buffer_size or full_buf_size >= self.server_info.buffer_size: return 0 - if buf_size > 1100: - return common.ord(os.urandom(1)[0]) % 128 - #self.pack_id - if buf_size > 400: - return struct.unpack('>H', os.urandom(2))[0] % 256 - else: - return struct.unpack('>H', os.urandom(2))[0] % 1024 + if buf_size > 900: + return struct.unpack('>H', os.urandom(2))[0] % rev_len + return self.trapezoid_random_int(rev_len, -0.3) def rnd_data(self, buf_size, full_buf_size): data_len = self.rnd_data_len(buf_size, full_buf_size) From 90ffd1c6ee836cfc441f2b56da4c7ea8d468c07d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 26 Mar 2017 18:29:13 +0800 Subject: [PATCH 392/487] display UID --- shadowsocks/tcprelay.py | 4 ++-- shadowsocks/udprelay.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7f01f712..817bfb27 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -573,9 +573,9 @@ def _handle_stage_addr(self, ogn_data, data): data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) connecttype, remote_addr, remote_port, header_length = header_result - common.connect_log('%s connecting %s:%d via port %d' % + common.connect_log('%s connecting %s:%d via port %d by UID %d' % ((connecttype == 0) and 'TCP' or 'UDP', - common.to_str(remote_addr), remote_port, self._server._listen_port)) + common.to_str(remote_addr), remote_port, self._server._listen_port, self._user_id)) self._remote_address = (common.to_str(remote_addr), remote_port) self._remote_udp = (connecttype != 0) # pause reading diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 3d8df3b3..19f47287 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -1229,9 +1229,13 @@ def _handle_server(self): logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) - common.connect_log('UDP data to %s:%d via port %d' % + if uid is None: + user_id = self._listen_port + else: + user_id = struct.unpack(' Date: Sun, 26 Mar 2017 19:07:30 +0800 Subject: [PATCH 393/487] display UID --- shadowsocks/tcprelay.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 817bfb27..99ccc4dd 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -463,7 +463,7 @@ def _get_redirect_host(self, client_address, ogn_data): return ("0.0.0.0", 0) def _handel_protocol_error(self, client_address, ogn_data): - logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d via port %d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1], self._server._listen_port)) + logging.warn("Protocol ERROR, TCP ogn data %s from %s:%d via port %d by UID %d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1], self._server._listen_port, self._user_id)) self._encrypt_correct = False #create redirect or disconnect by hash code host, port = self._get_redirect_host(client_address, ogn_data) @@ -644,15 +644,15 @@ def _create_remote_socket(self, ip, port): if self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: if self._remote_address: - raise Exception('IP %s is in forbidden list, when connect to %s:%d via port %d' % - (common.to_str(sa[0]), self._remote_address[0], self._remote_address[1], self._server._listen_port)) + raise Exception('IP %s is in forbidden list, when connect to %s:%d via port %d by UID %d' % + (common.to_str(sa[0]), self._remote_address[0], self._remote_address[1], self._server._listen_port, self._user_id)) raise Exception('IP %s is in forbidden list, reject' % common.to_str(sa[0])) if self._forbidden_portset: if sa[1] in self._forbidden_portset: if self._remote_address: - raise Exception('Port %d is in forbidden list, when connect to %s:%d via port %d' % - (sa[1], self._remote_address[0], self._remote_address[1], self._server._listen_port)) + raise Exception('Port %d is in forbidden list, when connect to %s:%d via port %d by UID %d' % + (sa[1], self._remote_address[0], self._remote_address[1], self._server._listen_port, self._user_id)) raise Exception('Port %d is in forbidden list, reject' % sa[1]) remote_sock = socket.socket(af, socktype, proto) self._remote_sock = remote_sock From 8e27035851b147c1e75dee0337ed133d3c139501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 26 Mar 2017 23:42:32 +0800 Subject: [PATCH 394/487] fix buffer_size --- shadowsocks/tcprelay.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 99ccc4dd..2d792bad 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -166,6 +166,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.key = self._encryptor.cipher_key server_info.head_len = 30 server_info.tcp_mss = 1448 + server_info.buffer_size = BUF_SIZE self._obfs.set_server_info(server_info) self._protocol = obfs.obfs(config['protocol']) From d82c931f8ec013b9e53ccf19e11da69b671a2150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 5 Apr 2017 21:46:19 +0800 Subject: [PATCH 395/487] auto adjust TCP_MAXSEG --- shadowsocks/obfs.py | 6 +++ shadowsocks/obfsplugin/auth.py | 20 +++++++++- shadowsocks/obfsplugin/obfs_tls.py | 5 +++ shadowsocks/obfsplugin/plain.py | 3 ++ shadowsocks/tcprelay.py | 63 +++++++++++++++++++++++------- shadowsocks/udprelay.py | 5 +-- 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 68a92f14..d89e5343 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -53,6 +53,9 @@ def init_data(self): def set_server_info(self, server_info): return self.obfs.set_server_info(server_info) + def get_server_info(self): + return self.obfs.get_server_info() + def get_method_info(self, method): method = method.lower() m = method_supported.get(method) @@ -62,6 +65,9 @@ def get_obfs(self, method): m = self._method_info return m[0](method) + def get_overhead(self, direction): + return self.obfs.get_overhead(direction) + def client_pre_encrypt(self, buf): return self.obfs.client_pre_encrypt(buf) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 98a958dc..7d8fe65a 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -82,10 +82,14 @@ def __init__(self, method): super(auth_base, self).__init__(method) self.method = method self.no_compatible_method = '' + self.overhead = 7 def init_data(self): return '' + def get_overhead(self, direction): # direction: true for c->s false for s->c + return self.overhead + def set_server_info(self, server_info): self.server_info = server_info @@ -103,6 +107,7 @@ def server_decode(self, buf): def not_match_return(self, buf): self.raw_trans = True + self.overhead = 0 if self.method == self.no_compatible_method: return (b'E'*2048, False) return (buf, False) @@ -871,6 +876,9 @@ def __init__(self, method): def init_data(self): return obfs_auth_v2_data() + def get_overhead(self, direction): # direction: true for c->s false for s->c + return 9 + def set_server_info(self, server_info): self.server_info = server_info try: @@ -1174,10 +1182,14 @@ def __init__(self, method, hashfunc): self.user_id = None self.user_key = None self.last_rnd_len = 0 + self.overhead = 9 def init_data(self): return obfs_auth_mu_data() + def get_overhead(self, direction): # direction: true for c->s false for s->c + return self.overhead + def set_server_info(self, server_info): self.server_info = server_info try: @@ -1198,9 +1210,15 @@ def trapezoid_random_int(self, max_val, d): return int(v * max_val) def rnd_data_len(self, buf_size, full_buf_size): + if full_buf_size >= self.server_info.buffer_size: + return 0 rev_len = self.server_info.tcp_mss - buf_size - 9 - if rev_len <= 0 or self.last_rnd_len >= self.server_info.buffer_size or full_buf_size >= self.server_info.buffer_size: + if rev_len == 0: return 0 + if rev_len < 0: + if rev_len > -self.server_info.tcp_mss: + return self.trapezoid_random_int(rev_len + self.server_info.tcp_mss, -0.3) + return common.ord(os.urandom(1)[0]) % 32 if buf_size > 900: return struct.unpack('>H', os.urandom(2))[0] % rev_len return self.trapezoid_random_int(rev_len, -0.3) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 7d35f7cc..fee3c810 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -65,10 +65,14 @@ def __init__(self, method): self.client_id = b'' self.max_time_dif = 60 * 60 * 24 # time dif (second) setting self.tls_version = b'\x03\x03' + self.overhead = 5 def init_data(self): return obfs_auth_data() + def get_overhead(self, direction): # direction: true for c->s false for s->c + return self.overhead + def sni(self, url): url = common.to_bytes(url) data = b"\x00" + struct.pack('>H', len(url)) + url @@ -184,6 +188,7 @@ def server_encode(self, buf): def decode_error_return(self, buf): self.handshake_status = -1 + self.overhead = 0 if self.method == 'tls1.2_ticket_auth': return (b'E'*2048, False, False) return (buf, True, False) diff --git a/shadowsocks/obfsplugin/plain.py b/shadowsocks/obfsplugin/plain.py index 1d4fbabd..2b943ac2 100644 --- a/shadowsocks/obfsplugin/plain.py +++ b/shadowsocks/obfsplugin/plain.py @@ -40,6 +40,9 @@ def __init__(self, method): def init_data(self): return b'' + def get_overhead(self, direction): # direction: true for c->s false for s->c + return 0 + def get_server_info(self): return self.server_info diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 2d792bad..ce9db081 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -91,6 +91,8 @@ WAIT_STATUS_WRITING = 2 WAIT_STATUS_READWRITING = WAIT_STATUS_READING | WAIT_STATUS_WRITING +NETWORK_MTU = 1492 +TCP_MSS = NETWORK_MTU - 40 BUF_SIZE = 32 * 1024 UDP_MAX_BUF_SIZE = 65536 @@ -137,6 +139,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._accept_address = local_sock.getsockname()[:2] self._user = None self._user_id = server._listen_port + self._tcp_mss = TCP_MSS # TCP Relay works as either sslocal or ssserver # if is_local, this is sslocal @@ -151,6 +154,16 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, return self._encrypt_correct = True self._obfs = obfs.obfs(config['obfs']) + self._protocol = obfs.obfs(config['protocol']) + self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._recv_buffer_size = BUF_SIZE - self._overhead + + try: + self._tcp_mss = local_sock.getsockopt(socket.SOL_TCP, socket.TCP_MAXSEG) + logging.debug("TCP MSS = %d" % (self._tcp_mss,)) + except: + pass + server_info = obfs.server_info(server.obfs_data) server_info.host = config['server'] server_info.port = server._listen_port @@ -165,11 +178,10 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.key_str = common.to_bytes(config['password']) server_info.key = self._encryptor.cipher_key server_info.head_len = 30 - server_info.tcp_mss = 1448 - server_info.buffer_size = BUF_SIZE + server_info.tcp_mss = self._tcp_mss + server_info.buffer_size = self._recv_buffer_size self._obfs.set_server_info(server_info) - self._protocol = obfs.obfs(config['protocol']) server_info = obfs.server_info(server.protocol_data) server_info.host = config['server'] server_info.port = server._listen_port @@ -184,8 +196,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, server_info.key_str = common.to_bytes(config['password']) server_info.key = self._encryptor.cipher_key server_info.head_len = 30 - server_info.tcp_mss = 1448 - server_info.buffer_size = BUF_SIZE + server_info.tcp_mss = self._tcp_mss + server_info.buffer_size = self._recv_buffer_size self._protocol.set_server_info(server_info) self._redir_list = config.get('redirect', ["*#0.0.0.0:0"]) @@ -439,18 +451,16 @@ def _get_redirect_host(self, client_address, ogn_data): items_sum = common.to_str(host).rsplit('#', 1) items_match = common.to_str(items_sum[0]).rsplit(':', 1) items = common.to_str(items_sum[1]).rsplit(':', 1) - if len(items_match) > 1: - if self._server._listen_port != int(items_match[1]): - continue - match_port = 0 if len(items_match) > 1: if items_match[1] != "*": try: - match_port = int(items_match[1]) + if self._server._listen_port != int(items_match[1]) and int(items_match[1]) != 0: + continue except: pass - if items_match[0] != "*" and common.match_regex(items_match[0], ogn_data) == False and \ - not (match_port == self._server._listen_port or match_port == 0): + + if items_match[0] != "*" and common.match_regex( + items_match[0], ogn_data) == False: continue if len(items) > 1: try: @@ -573,6 +583,12 @@ def _handle_stage_addr(self, ogn_data, data): if header_result is None: data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) + self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._recv_buffer_size = BUF_SIZE - self._overhead + server_info = self._obfs.get_server_info() + server_info.buffer_size = self._recv_buffer_size + server_info = self._protocol.get_server_info() + server_info.buffer_size = self._recv_buffer_size connecttype, remote_addr, remote_port, header_length = header_result common.connect_log('%s connecting %s:%d via port %d by UID %d' % ((connecttype == 0) and 'TCP' or 'UDP', @@ -743,15 +759,30 @@ def _handle_dns_resolved(self, result, error): logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() + def _get_read_size(self, sock, recv_buffer_size): + if self._overhead == 0: + return buffer_size + buffer_size = len(sock.recv(recv_buffer_size, socket.MSG_PEEK)) + if buffer_size == recv_buffer_size: + return buffer_size + s = buffer_size % self._tcp_mss + self._overhead + if s > self._tcp_mss: + return buffer_size + s - self._tcp_mss + return buffer_size + def _on_local_read(self): # handle all local read events and dispatch them to methods for # each stage if not self._local_sock: return is_local = self._is_local + if is_local: + recv_buffer_size = self._get_read_size(self._local_sock, self._recv_buffer_size) + else: + recv_buffer_size = BUF_SIZE data = None try: - data = self._local_sock.recv(BUF_SIZE) + data = self._local_sock.recv(recv_buffer_size) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) in \ (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): @@ -848,7 +879,11 @@ def _on_remote_read(self, is_remote_sock): data = struct.pack('>H', size) + data #logging.info('UDP over TCP recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) else: - data = self._remote_sock.recv(BUF_SIZE) + if self._is_local: + recv_buffer_size = BUF_SIZE + else: + recv_buffer_size = self._get_read_size(self._remote_sock, self._recv_buffer_size) + data = self._remote_sock.recv(recv_buffer_size) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) in \ (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK, 10035): #errno.WSAEWOULDBLOCK diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 19f47287..1b4aae96 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -75,9 +75,6 @@ from shadowsocks import encrypt, obfs, eventloop, lru_cache, common, shell from shadowsocks.common import pre_parse_header, parse_header, pack_addr -# we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time -TIMEOUTS_CLEAN_SIZE = 512 - # for each handler, we have 2 stream directions: # upstream: from client to server direction # read local and write to remote @@ -927,7 +924,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun server_info.key_str = common.to_bytes(config['password']) server_info.key = encrypt.encrypt_key(self._password, self._method) server_info.head_len = 30 - server_info.tcp_mss = 1440 + server_info.tcp_mss = 1452 server_info.buffer_size = BUF_SIZE self._protocol.set_server_info(server_info) From 862fa890ee77e617e5196b0c2a96545050e7095d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 5 Apr 2017 22:11:55 +0800 Subject: [PATCH 396/487] typo --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index ce9db081..a56ea1b3 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -761,7 +761,7 @@ def _handle_dns_resolved(self, result, error): def _get_read_size(self, sock, recv_buffer_size): if self._overhead == 0: - return buffer_size + return recv_buffer_size buffer_size = len(sock.recv(recv_buffer_size, socket.MSG_PEEK)) if buffer_size == recv_buffer_size: return buffer_size From 51c8ae4d7805b19542e39d42092ce4032752f837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 7 Apr 2017 11:55:06 +0800 Subject: [PATCH 397/487] bump --- CHANGES | 4 ++++ shadowsocks/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 05bda56b..00108900 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +3.1.2 2017-04-07 +- display UID +- auto adjust TCP MSS + 3.1.1 2017-03-25 - add "New session ticket" - ignore bind 10.0.0.0/8 and 192.168.0.0/16 by default diff --git a/shadowsocks/version.py b/shadowsocks/version.py index eed0e4ec..882c72e0 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.1.1 2017-03-25' + return '3.1.2 2017-04-07' From a3a8ed996c24157e47e404b4db53d1b0ed0893b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 8 Apr 2017 22:02:35 +0800 Subject: [PATCH 398/487] fix a mistake --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a56ea1b3..36167615 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -767,7 +767,7 @@ def _get_read_size(self, sock, recv_buffer_size): return buffer_size s = buffer_size % self._tcp_mss + self._overhead if s > self._tcp_mss: - return buffer_size + s - self._tcp_mss + return buffer_size - (s - self._tcp_mss) return buffer_size def _on_local_read(self): From 2b64802fedd11630075fbc968ed413e03991fa56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 9 Apr 2017 18:27:13 +0800 Subject: [PATCH 399/487] fix data length range --- shadowsocks/obfsplugin/auth.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 7d8fe65a..74959ad1 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1212,12 +1212,13 @@ def trapezoid_random_int(self, max_val, d): def rnd_data_len(self, buf_size, full_buf_size): if full_buf_size >= self.server_info.buffer_size: return 0 - rev_len = self.server_info.tcp_mss - buf_size - 9 + tcp_mss = self.server_info.tcp_mss if self.server_info.tcp_mss < 4096 else 4096 + rev_len = tcp_mss - buf_size - 9 if rev_len == 0: return 0 if rev_len < 0: - if rev_len > -self.server_info.tcp_mss: - return self.trapezoid_random_int(rev_len + self.server_info.tcp_mss, -0.3) + if rev_len > -tcp_mss: + return self.trapezoid_random_int(rev_len + tcp_mss, -0.3) return common.ord(os.urandom(1)[0]) % 32 if buf_size > 900: return struct.unpack('>H', os.urandom(2))[0] % rev_len From 1e7e707024858b97cd0ceed4729b2c1cabb0262e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 9 Apr 2017 18:33:29 +0800 Subject: [PATCH 400/487] set MSS up to 1500 --- shadowsocks/obfsplugin/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 74959ad1..3b9b739d 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -1212,7 +1212,7 @@ def trapezoid_random_int(self, max_val, d): def rnd_data_len(self, buf_size, full_buf_size): if full_buf_size >= self.server_info.buffer_size: return 0 - tcp_mss = self.server_info.tcp_mss if self.server_info.tcp_mss < 4096 else 4096 + tcp_mss = self.server_info.tcp_mss if self.server_info.tcp_mss < 1500 else 1500 rev_len = tcp_mss - buf_size - 9 if rev_len == 0: return 0 From 036df8aac6864bfae80915a4e7655c10bf2ff8a9 Mon Sep 17 00:00:00 2001 From: Handyman Date: Fri, 14 Apr 2017 02:37:32 -0700 Subject: [PATCH 401/487] Use thread.join() instead (#219) --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index a6541da7..ba863b68 100644 --- a/server.py +++ b/server.py @@ -55,7 +55,7 @@ def main(): thread.start() try: while thread.is_alive(): - time.sleep(10) + thread.join(10.0) except (KeyboardInterrupt, IOError, OSError) as e: import traceback traceback.print_exc() From 7deba0b995585b83f613e45089a80cc2830d1ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 22 Apr 2017 12:41:13 +0800 Subject: [PATCH 402/487] remove auth_aes128, fix bugs --- db_transfer.py | 5 +- shadowsocks/obfsplugin/auth.py | 263 ----------------------------- shadowsocks/obfsplugin/obfs_tls.py | 3 +- 3 files changed, 6 insertions(+), 265 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 5862de79..80d1d877 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -110,7 +110,10 @@ def del_server_out_of_bound_safe(self, last_rows, rows): merge_config_keys = ['password'] + read_config_keys for name in cfg.keys(): if hasattr(cfg[name], 'encode'): - cfg[name] = cfg[name].encode('utf-8') + try: + cfg[name] = cfg[name].encode('utf-8') + except Exception as e: + logging.warning('encode cfg key "%s" fail, val "%s"' % (name, cfg[name])) if port not in cur_servers: cur_servers[port] = passwd diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 3b9b739d..13c44114 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -46,9 +46,6 @@ def create_auth_sha1_v2(method): def create_auth_sha1_v4(method): return auth_sha1_v4(method) -def create_auth_aes128(method): - return auth_aes128(method) - def create_auth_aes128_md5(method): return auth_aes128_sha1(method, hashlib.md5) @@ -62,7 +59,6 @@ def create_auth_aes128_sha1(method): 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), 'auth_sha1_v4': (create_auth_sha1_v4,), 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), - 'auth_aes128': (create_auth_aes128,), 'auth_aes128_md5': (create_auth_aes128_md5,), 'auth_aes128_sha1': (create_auth_aes128_sha1,), } @@ -856,265 +852,6 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, sendback) -class auth_aes128(auth_base): - def __init__(self, method): - super(auth_aes128, self).__init__(method) - self.recv_buf = b'' - self.unit_len = 8100 - self.raw_trans = False - self.has_sent_header = False - self.has_recv_header = False - self.client_id = 0 - self.connection_id = 0 - self.max_time_dif = 60 * 60 * 24 # time dif (second) setting - self.salt = b"auth_aes128" - self.no_compatible_method = 'auth_aes128' - self.extra_wait_size = struct.unpack('>H', os.urandom(2))[0] % 1024 - self.pack_id = 0 - self.recv_id = 0 - - def init_data(self): - return obfs_auth_v2_data() - - def get_overhead(self, direction): # direction: true for c->s false for s->c - return 9 - - def set_server_info(self, server_info): - self.server_info = server_info - try: - max_client = int(server_info.protocol_param) - except: - max_client = 64 - self.server_info.data.set_max_client(max_client) - - def rnd_data(self, buf_size): - if buf_size > 1200: - return b'\x01' - - if self.pack_id > 4: - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 32) - elif buf_size > 900: - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) - else: - rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 512) - - if len(rnd_data) < 128: - return common.chr(len(rnd_data) + 1) + rnd_data - else: - return common.chr(255) + struct.pack(' 400: - rnd_len = common.ord(os.urandom(1)[0]) % 512 - else: - rnd_len = struct.unpack(' 0xFF000000: - self.server_info.data.local_client_id = b'' - if not self.server_info.data.local_client_id: - self.server_info.data.local_client_id = os.urandom(4) - logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) - self.server_info.data.connection_id = struct.unpack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def client_post_decrypt(self, buf): - if self.raw_trans: - return buf - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 4: - crc = struct.pack('= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data error') - if length > len(self.recv_buf): - break - - if struct.pack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def server_post_decrypt(self, buf): - if self.raw_trans: - return (buf, False) - self.recv_buf += buf - out_buf = b'' - sendback = False - - if not self.has_recv_header: - if len(self.recv_buf) < 30: - return (b'', False) - sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:20], hashlib.sha1).digest()[:10] - if sha1data != self.recv_buf[20:30]: - logging.error('auth_aes128 data uncorrect auth HMAC-SHA1 from %s:%d, data %s' % (self.server_info.client, self.server_info.client_port, binascii.hexlify(self.recv_buf))) - if len(self.recv_buf) < 30 + self.extra_wait_size: - return (b'', False) - return self.not_match_return(self.recv_buf) - - user_key = self.recv_buf[:4] - encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(user_key + self.server_info.key)) + self.salt, 'aes-128-cbc') - head = encryptor.decrypt(b'\x00' * 16 + self.recv_buf[4:20] + b'\x00') # need an extra byte or recv empty - length = struct.unpack(' self.max_time_dif: - logging.info('auth_aes128: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(head),)) - return self.not_match_return(self.recv_buf) - elif self.server_info.data.insert(client_id, connection_id): - self.has_recv_header = True - out_buf = self.recv_buf[30 + rnd_len:length - 4] - self.client_id = client_id - self.connection_id = connection_id - else: - logging.info('auth_aes128: auth fail, data %s' % (binascii.hexlify(out_buf),)) - return self.not_match_return(self.recv_buf) - self.recv_buf = self.recv_buf[length:] - self.has_recv_header = True - sendback = True - - while len(self.recv_buf) > 4: - crc = struct.pack('= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.recv_id == 0: - logging.info('auth_aes128: over size') - return (b'E'*2048, False) - else: - raise Exception('server_post_decrype data error') - if length > len(self.recv_buf): - break - - if struct.pack(' 0: ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf return ret - self.send_buffer += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf + if len(buf) > 0: + self.send_buffer += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf if self.handshake_status == 0: self.handshake_status = 1 data = self.tls_version + self.pack_auth_data(self.server_info.data.client_id) + b"\x20" + self.server_info.data.client_id + binascii.unhexlify(b"001cc02bc02fcca9cca8cc14cc13c00ac014c009c013009c0035002f000a" + b"0100") From db6629a1307ed38eb507bd926ad1503e90bc4eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 22 Apr 2017 12:41:44 +0800 Subject: [PATCH 403/487] test new protocol --- shadowsocks/obfs.py | 3 +- shadowsocks/obfsplugin/auth_chain.py | 587 +++++++++++++++++++++++++++ 2 files changed, 589 insertions(+), 1 deletion(-) create mode 100644 shadowsocks/obfsplugin/auth_chain.py diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index d89e5343..46867d88 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -23,7 +23,7 @@ import logging from shadowsocks import common -from shadowsocks.obfsplugin import plain, http_simple, obfs_tls, verify, auth +from shadowsocks.obfsplugin import plain, http_simple, obfs_tls, verify, auth, auth_chain method_supported = {} @@ -32,6 +32,7 @@ method_supported.update(obfs_tls.obfs_map) method_supported.update(verify.obfs_map) method_supported.update(auth.obfs_map) +method_supported.update(auth_chain.obfs_map) class server_info(object): def __init__(self, data): diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py new file mode 100644 index 00000000..2bb21da4 --- /dev/null +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python +# +# Copyright 2015-2015 breakwa11 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import hashlib +import logging +import binascii +import base64 +import time +import datetime +import random +import math +import struct +import zlib +import hmac +import hashlib + +import shadowsocks +from shadowsocks import common, lru_cache, encrypt +from shadowsocks.obfsplugin import plain +from shadowsocks.common import to_bytes, to_str, ord, chr + +def create_auth_chain_a(method): + return auth_chain_a(method) + + +obfs_map = { + 'auth_chain_a': (create_auth_chain_a,), +} + +class xorshift128plus(object): + max_int = (1 << 64) - 1 + mov_mask = (1 << (64 - 23)) - 1 + + def __init__(self): + self.v0 = 0 + self.v1 = 0 + + def next(self): + x = self.v0 + y = self.v1 + self.v0 = y + x ^= ((x & xorshift128plus.mov_mask) << 23) + x ^= (y ^ (x >> 17) ^ (y >> 26)) & xorshift128plus.max_int + self.v1 = x + return (x + y) & xorshift128plus.max_int + + def init_from_bin(self, bin): + bin += b'\0' * 16 + self.v0 = struct.unpack('= len(str2): + if str1[:len(str2)] == str2: + return True + return False + +class auth_base(plain.plain): + def __init__(self, method): + super(auth_base, self).__init__(method) + self.method = method + self.no_compatible_method = '' + self.overhead = 4 + + def init_data(self): + return '' + + def get_overhead(self, direction): # direction: true for c->s false for s->c + return self.overhead + + def set_server_info(self, server_info): + self.server_info = server_info + + def client_encode(self, buf): + return buf + + def client_decode(self, buf): + return (buf, False) + + def server_encode(self, buf): + return buf + + def server_decode(self, buf): + return (buf, True, False) + + def not_match_return(self, buf): + self.raw_trans = True + self.overhead = 0 + if self.method == self.no_compatible_method: + return (b'E'*2048, False) + return (buf, False) + +class client_queue(object): + def __init__(self, begin_id): + self.front = begin_id - 64 + self.back = begin_id + 1 + self.alloc = {} + self.enable = True + self.last_update = time.time() + self.ref = 0 + + def update(self): + self.last_update = time.time() + + def addref(self): + self.ref += 1 + + def delref(self): + if self.ref > 0: + self.ref -= 1 + + def is_active(self): + return (self.ref > 0) and (time.time() - self.last_update < 60 * 10) + + def re_enable(self, connection_id): + self.enable = True + self.front = connection_id - 64 + self.back = connection_id + 1 + self.alloc = {} + + def insert(self, connection_id): + if not self.enable: + logging.warn('obfs auth: not enable') + return False + if not self.is_active(): + self.re_enable(connection_id) + self.update() + if connection_id < self.front: + logging.warn('obfs auth: deprecated id, someone replay attack') + return False + if connection_id > self.front + 0x4000: + logging.warn('obfs auth: wrong id') + return False + if connection_id in self.alloc: + logging.warn('obfs auth: duplicate id, someone replay attack') + return False + if self.back <= connection_id: + self.back = connection_id + 1 + self.alloc[connection_id] = 1 + while (self.front in self.alloc) or self.front + 0x1000 < self.back: + if self.front in self.alloc: + del self.alloc[self.front] + self.front += 1 + self.addref() + return True + +class obfs_auth_chain_data(object): + def __init__(self, name): + self.name = name + self.user_id = {} + self.local_client_id = b'' + self.connection_id = 0 + self.set_max_client(64) # max active client count + + def update(self, user_id, client_id, connection_id): + if user_id not in self.user_id: + self.user_id[user_id] = lru_cache.LRUCache() + local_client_id = self.user_id[user_id] + + if client_id in local_client_id: + local_client_id[client_id].update() + + def set_max_client(self, max_client): + self.max_client = max_client + self.max_buffer = max(self.max_client * 2, 1024) + + def insert(self, user_id, client_id, connection_id): + if user_id not in self.user_id: + self.user_id[user_id] = lru_cache.LRUCache() + local_client_id = self.user_id[user_id] + + if local_client_id.get(client_id, None) is None or not local_client_id[client_id].enable: + if local_client_id.first() is None or len(local_client_id) < self.max_client: + if client_id not in local_client_id: + #TODO: check + local_client_id[client_id] = client_queue(connection_id) + else: + local_client_id[client_id].re_enable(connection_id) + return local_client_id[client_id].insert(connection_id) + + if not local_client_id[local_client_id.first()].is_active(): + del local_client_id[local_client_id.first()] + if client_id not in local_client_id: + #TODO: check + local_client_id[client_id] = client_queue(connection_id) + else: + local_client_id[client_id].re_enable(connection_id) + return local_client_id[client_id].insert(connection_id) + + logging.warn(self.name + ': no inactive client') + return False + else: + return local_client_id[client_id].insert(connection_id) + + def remove(self, user_id, client_id): + if user_id in self.user_id: + local_client_id = self.user_id[user_id] + if client_id in local_client_id: + local_client_id[client_id].delref() + +class auth_chain_a(auth_base): + def __init__(self, method): + super(auth_chain_a, self).__init__(method) + self.hashfunc = hashlib.md5 + self.recv_buf = b'' + self.unit_len = 2800 + self.raw_trans = False + self.has_sent_header = False + self.has_recv_header = False + self.client_id = 0 + self.connection_id = 0 + self.max_time_dif = 60 * 60 * 24 # time dif (second) setting + self.salt = b"auth_chain_a" + self.no_compatible_method = 'auth_chain_a' + self.pack_id = 1 + self.recv_id = 1 + self.user_id = None + self.user_id_num = 0 + self.user_key = None + self.overhead = 4 + self.client_over_head = 4 + self.last_client_hash = b'' + self.last_server_hash = b'' + self.random_client = xorshift128plus() + self.random_server = xorshift128plus() + self.encryptor = None + + def init_data(self): + return obfs_auth_chain_data(self.method) + + def get_overhead(self, direction): # direction: true for c->s false for s->c + return self.overhead + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param.split('#')[0]) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + + def trapezoid_random_float(self, d): + if d == 0: + return random.random() + s = random.random() + a = 1 - d + return (math.sqrt(a * a + 4 * d * s) - a) / (2 * d) + + def trapezoid_random_int(self, max_val, d): + v = self.trapezoid_random_float(d) + return int(v * max_val) + + def rnd_data_len(self, buf_size, last_hash, random): + if buf_size > 1440: + return 0 + random.init_from_bin_len(last_hash, buf_size) + if buf_size > 1300: + return random.next() % 31 + if buf_size > 900: + return random.next() % 127 + if buf_size > 400: + return random.next() % 521 + return random.next() % 1021 + + def rnd_start_pos(self, rand_len, random): + if rand_len > 0: + return random.next() % 8589934609 % rand_len + return 0 + + def rnd_data(self, buf_size, buf, last_hash, random): + rand_len = self.rnd_data_len(buf_size, last_hash, random) + + rnd_data_buf = os.urandom(rand_len) + + if buf_size == 0: + return rnd_data_buf + else: + if rand_len > 0: + start_pos = self.rnd_start_pos(rand_len, random) + return rnd_data_buf[:start_pos] + buf + rnd_data_buf[start_pos:] + else: + return buf + + def pack_client_data(self, buf): + data = self.rnd_data(len(buf), buf, self.last_client_hash, self.random_client) + data_len = len(data) + 8 + mac_key = self.user_key + struct.pack(' 400: + rnd_len = struct.unpack(' 0xFF000000: + self.server_info.data.local_client_id = b'' + if not self.server_info.data.local_client_id: + self.server_info.data.local_client_id = os.urandom(4) + logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) + self.server_info.data.connection_id = struct.unpack(' self.unit_len: + ret += self.pack_client_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_client_data(buf) + return ret + + def client_post_decrypt(self, buf): + if self.raw_trans: + return buf + self.recv_buf += buf + out_buf = b'' + while len(self.recv_buf) > 4: + mac_key = self.user_key + struct.pack('= 8192 or length < 7: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data error') + if length > len(self.recv_buf): + break + + if hmac.new(mac_key, self.recv_buf[:length - 4], self.hashfunc).digest()[:4] != self.recv_buf[length - 4:length]: + self.raw_trans = True + self.recv_buf = b'' + raise Exception('client_post_decrypt data uncorrect checksum') + + self.recv_id = (self.recv_id + 1) & 0xFFFFFFFF + pos = common.ord(self.recv_buf[4]) + if pos < 255: + pos += 4 + else: + pos = struct.unpack(' self.unit_len: + ret += self.pack_server_data(buf[:self.unit_len]) + buf = buf[self.unit_len:] + ret += self.pack_server_data(buf) + return ret + + def server_post_decrypt(self, buf): + if self.raw_trans: + return (buf, False) + self.recv_buf += buf + out_buf = b'' + sendback = False + + if not self.has_recv_header: + if len(self.recv_buf) >= 12 or len(self.recv_buf) in [7, 8]: + recv_len = min(len(self.recv_buf), 12) + mac_key = self.server_info.recv_iv + self.server_info.key + md5data = hmac.new(mac_key, self.recv_buf[:4], self.hashfunc).digest() + if md5data[:recv_len - 4] != self.recv_buf[4:recv_len]: + return self.not_match_return(self.recv_buf) + + if len(self.recv_buf) < 12 + 24: + return (b'', False) + + self.last_client_hash = md5data + md5data = hmac.new(mac_key, self.recv_buf[12 : 12 + 20], self.hashfunc).digest() + if md5data[:4] != self.recv_buf[32:36]: + logging.error('%s data uncorrect auth HMAC-SHA1 from %s:%d, data %s' % (self.no_compatible_method, self.server_info.client, self.server_info.client_port, binascii.hexlify(self.recv_buf))) + if len(self.recv_buf) < 36: + return (b'', False) + return self.not_match_return(self.recv_buf) + + self.last_server_hash = md5data + uid = struct.unpack(' self.max_time_dif: + logging.info('%s: wrong timestamp, time_dif %d, data %s' % (self.no_compatible_method, time_dif, binascii.hexlify(head))) + return self.not_match_return(self.recv_buf) + elif self.server_info.data.insert(self.user_id, client_id, connection_id): + self.has_recv_header = True + self.client_id = client_id + self.connection_id = connection_id + else: + logging.info('%s: auth fail, data %s' % (self.no_compatible_method, binascii.hexlify(out_buf))) + return self.not_match_return(self.recv_buf) + + self.encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(self.user_key)) + to_bytes(base64.b64encode(self.last_server_hash)), 'rc4') + self.recv_buf = self.recv_buf[36:] + self.has_recv_header = True + sendback = True + + while len(self.recv_buf) > 4: + mac_key = self.user_key + struct.pack('= 4096: + self.raw_trans = True + self.recv_buf = b'' + if self.recv_id == 0: + logging.info(self.no_compatible_method + ': over size') + return (b'E'*2048, False) + else: + raise Exception('server_post_decrype data error') + + if length + 4 > len(self.recv_buf): + break + + client_hash = hmac.new(mac_key, self.recv_buf[:length + 2], self.hashfunc).digest() + if client_hash[:2] != self.recv_buf[length + 2 : length + 4]: + logging.info('%s: checksum error, data %s' % (self.no_compatible_method, binascii.hexlify(self.recv_buf[:length]))) + self.raw_trans = True + self.recv_buf = b'' + if self.recv_id == 0: + return (b'E'*2048, False) + else: + raise Exception('server_post_decrype data uncorrect checksum') + + self.recv_id = (self.recv_id + 1) & 0xFFFFFFFF + pos = 2 + if data_len > 0 and rand_len > 0: + pos = 2 + self.rnd_start_pos(rand_len, self.random_client) + out_buf += self.encryptor.decrypt(self.recv_buf[pos : data_len + pos]) + self.last_client_hash = client_hash + self.recv_buf = self.recv_buf[length + 4:] + if data_len == 0: + sendback = True + + if out_buf: + self.server_info.data.update(self.user_id, self.client_id, self.connection_id) + return (out_buf, sendback) + + def client_udp_pre_encrypt(self, buf): + if self.user_key is None: + if b':' in to_bytes(self.server_info.protocol_param): + try: + items = to_bytes(self.server_info.protocol_param).split(':') + self.user_key = self.hashfunc(items[1]).digest() + self.user_id = struct.pack(' Date: Tue, 25 Apr 2017 10:27:51 +0800 Subject: [PATCH 404/487] remove auth_sha1, auth_sha1_v2, verify_simple, auth_simple, verify_sha1 remove TCP over UDP add UDP part of auth_chain_a --- db_transfer.py | 4 +- shadowsocks/obfs.py | 7 +- shadowsocks/obfsplugin/auth.py | 450 +------------- shadowsocks/obfsplugin/auth_chain.py | 84 ++- shadowsocks/obfsplugin/plain.py | 2 +- shadowsocks/obfsplugin/verify.py | 206 ------- shadowsocks/tcprelay.py | 2 +- shadowsocks/udprelay.py | 837 +-------------------------- 8 files changed, 87 insertions(+), 1505 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 80d1d877..a5fb8623 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -6,7 +6,7 @@ import sys from server_pool import ServerPool import traceback -from shadowsocks import common, shell, lru_cache +from shadowsocks import common, shell, lru_cache, obfs from configloader import load_config, get_config import importloader @@ -123,7 +123,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): if allow: allow_users[port] = passwd - if 'protocol' in cfg and 'protocol_param' in cfg and common.to_str(cfg['protocol']) in ['auth_aes128_md5', 'auth_aes128_sha1']: + if 'protocol' in cfg and 'protocol_param' in cfg and common.to_str(cfg['protocol']) in obfs.mu_protocol(): if '#' in common.to_str(cfg['protocol_param']): mu_servers[port] = passwd del allow_users[port] diff --git a/shadowsocks/obfs.py b/shadowsocks/obfs.py index 46867d88..3dfdb141 100644 --- a/shadowsocks/obfs.py +++ b/shadowsocks/obfs.py @@ -34,6 +34,9 @@ method_supported.update(auth.obfs_map) method_supported.update(auth_chain.obfs_map) +def mu_protocol(): + return ["auth_aes128_md5", "auth_aes128_sha1", "auth_chain_a"] + class server_info(object): def __init__(self, data): self.data = data @@ -99,8 +102,8 @@ def client_udp_pre_encrypt(self, buf): def client_udp_post_decrypt(self, buf): return self.obfs.client_udp_post_decrypt(buf) - def server_udp_pre_encrypt(self, buf): - return self.obfs.server_udp_pre_encrypt(buf) + def server_udp_pre_encrypt(self, buf, uid): + return self.obfs.server_udp_pre_encrypt(buf, uid) def server_udp_post_decrypt(self, buf): return self.obfs.server_udp_post_decrypt(buf) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 13c44114..51a1b00c 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -37,12 +37,6 @@ from shadowsocks.obfsplugin import plain from shadowsocks.common import to_bytes, to_str, ord, chr -def create_auth_sha1(method): - return auth_sha1(method) - -def create_auth_sha1_v2(method): - return auth_sha1_v2(method) - def create_auth_sha1_v4(method): return auth_sha1_v4(method) @@ -53,10 +47,6 @@ def create_auth_aes128_sha1(method): return auth_aes128_sha1(method, hashlib.sha1) obfs_map = { - 'auth_sha1': (create_auth_sha1,), - 'auth_sha1_compatible': (create_auth_sha1,), - 'auth_sha1_v2': (create_auth_sha1_v2,), - 'auth_sha1_v2_compatible': (create_auth_sha1_v2,), 'auth_sha1_v4': (create_auth_sha1_v4,), 'auth_sha1_v4_compatible': (create_auth_sha1_v4,), 'auth_aes128_md5': (create_auth_aes128_md5,), @@ -69,10 +59,6 @@ def match_begin(str1, str2): return True return False -class obfs_verify_data(object): - def __init__(self): - pass - class auth_base(plain.plain): def __init__(self, method): super(auth_base, self).__init__(method) @@ -153,239 +139,6 @@ def insert(self, connection_id): self.front += 1 return True -class obfs_auth_data(object): - def __init__(self): - self.client_id = {} - self.startup_time = int(time.time() - 30) & 0xFFFFFFFF - self.local_client_id = b'' - self.connection_id = 0 - self.set_max_client(64) # max active client count - - def update(self, client_id, connection_id): - if client_id in self.client_id: - self.client_id[client_id].update() - - def set_max_client(self, max_client): - self.max_client = max_client - self.max_buffer = max(self.max_client * 2, 256) - - def insert(self, client_id, connection_id): - if client_id not in self.client_id or not self.client_id[client_id].enable: - active = 0 - for c_id in self.client_id: - if self.client_id[c_id].is_active(): - active += 1 - if active >= self.max_client: - logging.warn('obfs auth: max active clients exceeded') - return False - - if len(self.client_id) < self.max_client: - if client_id not in self.client_id: - self.client_id[client_id] = client_queue(connection_id) - else: - self.client_id[client_id].re_enable(connection_id) - return self.client_id[client_id].insert(connection_id) - keys = self.client_id.keys() - random.shuffle(keys) - for c_id in keys: - if not self.client_id[c_id].is_active() and self.client_id[c_id].enable: - if len(self.client_id) >= self.max_buffer: - del self.client_id[c_id] - else: - self.client_id[c_id].enable = False - if client_id not in self.client_id: - self.client_id[client_id] = client_queue(connection_id) - else: - self.client_id[client_id].re_enable(connection_id) - return self.client_id[client_id].insert(connection_id) - logging.warn('obfs auth: no inactive client [assert]') - return False - else: - return self.client_id[client_id].insert(connection_id) - -class auth_sha1(auth_base): - def __init__(self, method): - super(auth_sha1, self).__init__(method) - self.recv_buf = b'' - self.unit_len = 8000 - self.decrypt_packet_num = 0 - self.raw_trans = False - self.has_sent_header = False - self.has_recv_header = False - self.client_id = 0 - self.connection_id = 0 - self.max_time_dif = 60 * 60 # time dif (second) setting - self.no_compatible_method = 'auth_sha1' - - def init_data(self): - return obfs_auth_data() - - def set_server_info(self, server_info): - self.server_info = server_info - try: - max_client = int(server_info.protocol_param) - except: - max_client = 64 - self.server_info.data.set_max_client(max_client) - - def pack_data(self, buf): - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16) - data = common.chr(len(rnd_data) + 1) + rnd_data + buf - data = struct.pack('>H', len(data) + 6) + data - adler32 = zlib.adler32(data) & 0xFFFFFFFF - data += struct.pack('H', len(data) + 16) + data - crc = binascii.crc32(self.server_info.key) & 0xFFFFFFFF - data = struct.pack(' 0xFF000000: - self.server_info.data.local_client_id = b'' - if not self.server_info.data.local_client_id: - self.server_info.data.local_client_id = os.urandom(4) - logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) - self.server_info.data.connection_id = struct.unpack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def client_post_decrypt(self, buf): - if self.raw_trans: - return buf - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data error') - if length > len(self.recv_buf): - break - - if struct.pack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def server_post_decrypt(self, buf): - if self.raw_trans: - return (buf, False) - self.recv_buf += buf - out_buf = b'' - if not self.has_recv_header: - if len(self.recv_buf) < 6: - return (b'', False) - crc = struct.pack('H', self.recv_buf[4:6])[0] - if length > 2048: - return self.not_match_return(self.recv_buf) - if length > len(self.recv_buf): - return (b'', False) - sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] - if sha1data != self.recv_buf[length - 10:length]: - logging.error('auth_sha1 data uncorrect auth HMAC-SHA1') - return self.not_match_return(self.recv_buf) - pos = common.ord(self.recv_buf[6]) + 6 - out_buf = self.recv_buf[pos:length - 10] - if len(out_buf) < 12: - logging.info('auth_sha1: too short, data %s' % (binascii.hexlify(self.recv_buf),)) - return self.not_match_return(self.recv_buf) - utc_time = struct.unpack(' self.max_time_dif \ - or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2: - logging.info('auth_sha1: wrong timestamp, time_dif %d, data %s' % (time_dif, binascii.hexlify(out_buf),)) - return self.not_match_return(self.recv_buf) - elif self.server_info.data.insert(client_id, connection_id): - self.has_recv_header = True - out_buf = out_buf[12:] - self.client_id = client_id - self.connection_id = connection_id - else: - logging.info('auth_sha1: auth fail, data %s' % (binascii.hexlify(out_buf),)) - return self.not_match_return(self.recv_buf) - self.recv_buf = self.recv_buf[length:] - self.has_recv_header = True - - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - logging.info('auth_sha1: over size') - return (b'E'*2048, False) - else: - raise Exception('server_post_decrype data error') - if length > len(self.recv_buf): - break - - if struct.pack(' 1300: - return b'\x01' - - if buf_size > 400: - rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 128) - return common.chr(len(rnd_data) + 1) + rnd_data - - rnd_data = os.urandom(struct.unpack('>H', os.urandom(2))[0] % 1024) - return common.chr(255) + struct.pack('>H', len(rnd_data) + 3) + rnd_data - - def pack_data(self, buf): - data = self.rnd_data(len(buf)) + buf - data = struct.pack('>H', len(data) + 6) + data - adler32 = zlib.adler32(data) & 0xFFFFFFFF - data += struct.pack('H', len(data) + 16) + data - crc = binascii.crc32(self.salt + self.server_info.key) & 0xFFFFFFFF - data = struct.pack(' 0xFF000000: - self.server_info.data.local_client_id = b'' - if not self.server_info.data.local_client_id: - self.server_info.data.local_client_id = os.urandom(8) - logging.debug("local_client_id %s" % (binascii.hexlify(self.server_info.data.local_client_id),)) - self.server_info.data.connection_id = struct.unpack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def client_post_decrypt(self, buf): - if self.raw_trans: - return buf - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data error') - if length > len(self.recv_buf): - break - - if struct.pack('H', self.recv_buf[3:5])[0] + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - - if out_buf: - self.decrypt_packet_num += 1 - return out_buf - - def server_pre_encrypt(self, buf): - if self.raw_trans: - return buf - ret = b'' - while len(buf) > self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def server_post_decrypt(self, buf): - if self.raw_trans: - return (buf, False) - self.recv_buf += buf - out_buf = b'' - sendback = False - - if not self.has_recv_header: - if len(self.recv_buf) < 6: - return (b'', False) - crc = struct.pack('H', self.recv_buf[4:6])[0] - if length > 2048: - return self.not_match_return(self.recv_buf) - if length > len(self.recv_buf): - return (b'', False) - sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:length - 10], hashlib.sha1).digest()[:10] - if sha1data != self.recv_buf[length - 10:length]: - logging.error('auth_sha1_v2 data uncorrect auth HMAC-SHA1') - return self.not_match_return(self.recv_buf) - pos = common.ord(self.recv_buf[6]) - if pos < 255: - pos += 6 - else: - pos = struct.unpack('>H', self.recv_buf[7:9])[0] + 6 - out_buf = self.recv_buf[pos:length - 10] - if len(out_buf) < 12: - logging.info('auth_sha1_v2: too short, data %s' % (binascii.hexlify(self.recv_buf),)) - return self.not_match_return(self.recv_buf) - client_id = struct.unpack(' 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - logging.info('auth_sha1_v2: over size') - return (b'E'*2048, False) - else: - raise Exception('server_post_decrype data error') - if length > len(self.recv_buf): - break - - if struct.pack('H', self.recv_buf[3:5])[0] + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - if pos == length - 4: - sendback = True - - if out_buf: - self.server_info.data.update(self.client_id, self.connection_id) - self.decrypt_packet_num += 1 - return (out_buf, sendback) - class auth_sha1_v4(auth_base): def __init__(self, method): super(auth_sha1_v4, self).__init__(method) @@ -1215,7 +767,7 @@ def client_udp_post_decrypt(self, buf): return b'' return buf[:-4] - def server_udp_pre_encrypt(self, buf): + def server_udp_pre_encrypt(self, buf, uid): user_key = self.server_info.key return buf + hmac.new(user_key, buf, self.hashfunc).digest()[:4] diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index 2bb21da4..ff0a812f 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -289,6 +289,10 @@ def rnd_data_len(self, buf_size, last_hash, random): return random.next() % 521 return random.next() % 1021 + def udp_rnd_data_len(self, last_hash, random): + random.init_from_bin(last_hash) + return random.next() % 127 + def rnd_start_pos(self, rand_len, random): if rand_len > 0: return random.next() % 8589934609 % rand_len @@ -457,20 +461,12 @@ def server_post_decrypt(self, buf): return (b'', False) self.last_client_hash = md5data - md5data = hmac.new(mac_key, self.recv_buf[12 : 12 + 20], self.hashfunc).digest() - if md5data[:4] != self.recv_buf[32:36]: - logging.error('%s data uncorrect auth HMAC-SHA1 from %s:%d, data %s' % (self.no_compatible_method, self.server_info.client, self.server_info.client_port, binascii.hexlify(self.recv_buf))) - if len(self.recv_buf) < 36: - return (b'', False) - return self.not_match_return(self.recv_buf) - - self.last_server_hash = md5data uid = struct.unpack('H', len(data) + 6) + data - crc = (0xffffffff - binascii.crc32(data)) & 0xffffffff - data += struct.pack(' self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def client_post_decrypt(self, buf): - if self.raw_trans: - return buf - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data error') - if length > len(self.recv_buf): - break - - if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: - self.raw_trans = True - self.recv_buf = b'' - raise Exception('client_post_decrypt data uncorrect CRC32') - - pos = common.ord(self.recv_buf[2]) + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - - if out_buf: - self.decrypt_packet_num += 1 - return out_buf - - def server_pre_encrypt(self, buf): - ret = b'' - while len(buf) > self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def server_post_decrypt(self, buf): - if self.raw_trans: - return (buf, False) - self.recv_buf += buf - out_buf = b'' - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] - if length >= 8192 or length < 7: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return (b'E'*2048, False) - else: - raise Exception('server_post_decrype data error') - if length > len(self.recv_buf): - break - - if (binascii.crc32(self.recv_buf[:length]) & 0xffffffff) != 0xffffffff: - self.raw_trans = True - self.recv_buf = b'' - if self.decrypt_packet_num == 0: - return (b'E'*2048, False) - else: - raise Exception('server_post_decrype data uncorrect CRC32') - - pos = common.ord(self.recv_buf[2]) + 2 - out_buf += self.recv_buf[pos:length - 4] - self.recv_buf = self.recv_buf[length:] - - if out_buf: - self.decrypt_packet_num += 1 - return (out_buf, False) - class verify_deflate(verify_base): def __init__(self, method): super(verify_deflate, self).__init__(method) @@ -258,103 +152,3 @@ def server_post_decrypt(self, buf): self.decrypt_packet_num += 1 return (out_buf, False) -class verify_sha1(verify_base): - def __init__(self, method): - super(verify_sha1, self).__init__(method) - self.recv_buf = b'' - self.unit_len = 8100 - self.raw_trans = False - self.pack_id = 0 - self.recv_id = 0 - self.has_sent_header = False - self.has_recv_header = False - - def pack_data(self, buf): - if len(buf) == 0: - return b'' - sha1data = hmac.new(self.server_info.iv + struct.pack('>I', self.pack_id), buf, hashlib.sha1).digest() - data = struct.pack('>H', len(buf)) + sha1data[:10] + buf - self.pack_id += 1 - return data - - def pack_auth_data(self, buf): - data = chr(ord(buf[0]) | 0x10) + buf[1:] - data += hmac.new(self.server_info.iv + self.server_info.key, data, hashlib.sha1).digest()[:10] - return data - - def client_pre_encrypt(self, buf): - ret = b'' - if not self.has_sent_header: - datalen = self.get_head_size(buf, 30) - ret += self.pack_auth_data(buf[:datalen]) - buf = buf[datalen:] - self.has_sent_header = True - while len(buf) > self.unit_len: - ret += self.pack_data(buf[:self.unit_len]) - buf = buf[self.unit_len:] - ret += self.pack_data(buf) - return ret - - def client_post_decrypt(self, buf): - return buf - - def server_pre_encrypt(self, buf): - return buf - - def not_match_return(self, buf): - self.raw_trans = True - if self.method == 'verify_sha1': - return (b'E'*2048, False) - return (buf, False) - - def server_post_decrypt(self, buf): - if self.raw_trans: - return (buf, False) - self.recv_buf += buf - out_buf = b'' - if not self.has_recv_header: - if len(self.recv_buf) < 2: - return (b'', False) - if (ord(self.recv_buf[0]) & 0x10) != 0x10: - return self.not_match_return(self.recv_buf) - head_size = self.get_head_size(self.recv_buf, 65536) - if len(self.recv_buf) < head_size + 10: - return self.not_match_return(self.recv_buf) - sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, self.recv_buf[:head_size], hashlib.sha1).digest()[:10] - if sha1data != self.recv_buf[head_size:head_size + 10]: - logging.error('server_post_decrype data uncorrect auth HMAC-SHA1') - return self.not_match_return(self.recv_buf) - out_buf = to_bytes(chr(ord(self.recv_buf[0]) & 0xEF)) + self.recv_buf[1:head_size] - self.recv_buf = self.recv_buf[head_size + 10:] - self.has_recv_header = True - while len(self.recv_buf) > 2: - length = struct.unpack('>H', self.recv_buf[:2])[0] + 12 - if length > len(self.recv_buf): - break - - data = self.recv_buf[12:length] - sha1data = hmac.new(self.server_info.recv_iv + struct.pack('>I', self.recv_id), data, hashlib.sha1).digest()[:10] - if sha1data != self.recv_buf[2:12]: - raise Exception('server_post_decrype data uncorrect chunk HMAC-SHA1') - - self.recv_id = (self.recv_id + 1) & 0xFFFFFFFF - out_buf += data - self.recv_buf = self.recv_buf[length:] - - return (out_buf, False) - - def client_udp_pre_encrypt(self, buf): - ret = self.pack_auth_data(buf) - return chr(ord(buf[0]) | 0x10) + buf[1:] - - def server_udp_post_decrypt(self, buf): - if buf and ((ord(buf[0]) & 0x10) == 0x10): - if len(buf) <= 11: - return (b'', None) - sha1data = hmac.new(self.server_info.recv_iv + self.server_info.key, buf[:-10], hashlib.sha1).digest()[:10] - if sha1data != buf[-10:]: - return (b'', None) - return (to_bytes(chr(ord(buf[0]) & 0xEF)) + buf[1:-10], None) - else: - return (buf, None) - diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 36167615..467c0f8f 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1116,7 +1116,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun listen_port = config['server_port'] self._listen_port = listen_port - if common.to_bytes(config['protocol']) in [b"auth_aes128_md5", b"auth_aes128_sha1"]: + if common.to_str(config['protocol']) in obfs.mu_protocol(): self._update_users(None, None) addrs = socket.getaddrinfo(listen_addr, listen_port, 0, diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 1b4aae96..98b8e83f 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -123,751 +123,6 @@ RSP_STATE_DISCONNECT = b"\x04" RSP_STATE_REDIRECT = b"\x05" -class UDPLocalAddress(object): - def __init__(self, addr): - self.addr = addr - self.last_activity = time.time() - - def is_timeout(self): - return time.time() - self.last_activity > 30 - -class PacketInfo(object): - def __init__(self, data): - self.data = data - self.time = time.time() - -class SendingQueue(object): - def __init__(self): - self.queue = {} - self.begin_id = 0 - self.end_id = 1 - self.interval = 0.5 - - def append(self, data): - self.queue[self.end_id] = PacketInfo(data) - self.end_id += 1 - return self.end_id - 1 - - def empty(self): - return self.begin_id + 1 == self.end_id - - def size(self): - return self.end_id - self.begin_id - 1 - - def get_begin_id(self): - return self.begin_id - - def get_end_id(self): - return self.end_id - - def get_data_list(self, pack_id_base, pack_id_list): - ret_list = [] - curtime = time.time() - for pack_id in pack_id_list: - offset = pack_id_base + pack_id - if offset <= self.begin_id or self.end_id <= offset: - continue - ret_data = self.queue[offset] - if curtime - ret_data.time > self.interval: - ret_data.time = curtime - ret_list.append( (offset, ret_data.data) ) - return ret_list - - def set_finish(self, begin_id, done_list): - while self.begin_id < begin_id: - self.begin_id += 1 - del self.queue[self.begin_id] - -class RecvQueue(object): - def __init__(self): - self.queue = {} - self.miss_queue = set() - self.begin_id = 0 - self.end_id = 1 - - def empty(self): - return self.begin_id + 1 == self.end_id - - def insert(self, pack_id, data): - if (pack_id not in self.queue) and pack_id > self.begin_id: - self.queue[pack_id] = PacketInfo(data) - if self.end_id == pack_id: - self.end_id = pack_id + 1 - elif self.end_id < pack_id: - eid = self.end_id - while eid < pack_id: - self.miss_queue.add(eid) - eid += 1 - self.end_id = pack_id + 1 - else: - self.miss_queue.remove(pack_id) - - def set_end(self, end_id): - if end_id > self.end_id: - eid = self.end_id - while eid < end_id: - self.miss_queue.add(eid) - eid += 1 - self.end_id = end_id - - def get_begin_id(self): - return self.begin_id - - def has_data(self): - return (self.begin_id + 1) in self.queue - - def get_data(self): - if (self.begin_id + 1) in self.queue: - self.begin_id += 1 - pack_id = self.begin_id - ret_data = self.queue[pack_id] - del self.queue[pack_id] - return (pack_id, ret_data.data) - - def get_missing_id(self, begin_id): - missing = [] - if begin_id == 0: - begin_id = self.begin_id - for i in self.miss_queue: - if i - begin_id > 32768: - break - missing.append(i - begin_id) - return (begin_id, missing) - -class AddressMap(object): - def __init__(self): - self._queue = [] - self._addr_map = {} - - def add(self, addr): - if addr in self._addr_map: - self._addr_map[addr] = UDPLocalAddress(addr) - else: - self._addr_map[addr] = UDPLocalAddress(addr) - self._queue.append(addr) - - def keys(self): - return self._queue - - def get(self): - if self._queue: - while True: - if len(self._queue) == 1: - return self._queue[0] - index = random.randint(0, len(self._queue) - 1) - addr = self._queue[index] - if self._addr_map[addr].is_timeout(): - self._queue[index] = self._queue[len(self._queue) - 1] - del self._queue[len(self._queue) - 1] - del self._addr_map[addr] - else: - break - return addr - else: - return None - -class TCPRelayHandler(object): - def __init__(self, server, reqid_to_handlers, fd_to_handlers, loop, - local_sock, local_id, client_param, config, - dns_resolver, is_local): - self._server = server - self._reqid_to_handlers = reqid_to_handlers - self._fd_to_handlers = fd_to_handlers - self._loop = loop - self._local_sock = local_sock - self._remote_sock = None - self._remote_udp = False - self._config = config - self._dns_resolver = dns_resolver - self._local_id = local_id - - self._is_local = is_local - self._stage = STAGE_INIT - self._password = config['password'] - self._method = config['method'] - self._fastopen_connected = False - self._data_to_write_to_local = [] - self._data_to_write_to_remote = [] - self._upstream_status = WAIT_STATUS_READING - self._downstream_status = WAIT_STATUS_INIT - self._request_id = 0 - self._client_address = AddressMap() - self._remote_address = None - self._sendingqueue = SendingQueue() - self._recvqueue = RecvQueue() - if 'forbidden_ip' in config: - self._forbidden_iplist = config['forbidden_ip'] - else: - self._forbidden_iplist = None - if 'forbidden_port' in config: - self._forbidden_portset = config['forbidden_port'] - else: - self._forbidden_portset = None - #fd_to_handlers[local_sock.fileno()] = self - #local_sock.setblocking(False) - #loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) - self.last_activity = 0 - self._update_activity() - self._random_mtu_size = [random.randint(POST_MTU_MIN, POST_MTU_MAX) for i in range(1024)] - self._random_mtu_index = 0 - - self._rand_data = b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" * 4 - - def __hash__(self): - # default __hash__ is id / 16 - # we want to eliminate collisions - return id(self) - - @property - def remote_address(self): - return self._remote_address - - def add_local_address(self, addr): - self._client_address.add(addr) - - def get_local_address(self): - return self._client_address.get() - - def _update_activity(self): - # tell the TCP Relay we have activities recently - # else it will think we are inactive and timed out - self._server.update_activity(self) - - def _update_stream(self, stream, status): - # update a stream to a new waiting status - - # check if status is changed - # only update if dirty - dirty = False - if stream == STREAM_DOWN: - if self._downstream_status != status: - self._downstream_status = status - dirty = True - elif stream == STREAM_UP: - if self._upstream_status != status: - self._upstream_status = status - dirty = True - if dirty: - ''' - if self._local_sock: - event = eventloop.POLL_ERR - if self._downstream_status & WAIT_STATUS_WRITING: - event |= eventloop.POLL_OUT - if self._upstream_status & WAIT_STATUS_READING: - event |= eventloop.POLL_IN - self._loop.modify(self._local_sock, event) - ''' - if self._remote_sock: - event = eventloop.POLL_ERR - if self._downstream_status & WAIT_STATUS_READING: - event |= eventloop.POLL_IN - if self._upstream_status & WAIT_STATUS_WRITING: - event |= eventloop.POLL_OUT - self._loop.modify(self._remote_sock, event) - - def _write_to_sock(self, data, sock, addr = None): - # write data to sock - # if only some of the data are written, put remaining in the buffer - # and update the stream to wait for writing - if not data or not sock: - return False - - uncomplete = False - retry = 0 - if sock == self._local_sock: - data = encrypt.encrypt_all(self._password, self._method, 1, data) - if addr is None: - return False - try: - self._server.write_to_server_socket(data, addr) - except (OSError, IOError) as e: - error_no = eventloop.errno_from_exception(e) - uncomplete = True - if error_no in (errno.EAGAIN, errno.EINPROGRESS, - errno.EWOULDBLOCK): - pass - else: - #traceback.print_exc() - shell.print_exception(e) - self.destroy() - return False - else: - try: - l = len(data) - s = sock.send(data) - if s < l: - data = data[s:] - uncomplete = True - except (OSError, IOError) as e: - error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS, - errno.EWOULDBLOCK): - uncomplete = True - else: - #logging.error(traceback.extract_stack()) - #traceback.print_exc() - shell.print_exception(e) - self.destroy() - return False - if uncomplete: - if sock == self._local_sock: - self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) - elif sock == self._remote_sock: - self._data_to_write_to_remote.append(data) - self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) - else: - logging.error('write_all_to_sock:unknown socket') - else: - if sock == self._local_sock: - if self._sendingqueue.size() > SENDING_WINDOW_SIZE: - self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) - else: - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) - elif sock == self._remote_sock: - self._update_stream(STREAM_UP, WAIT_STATUS_READING) - else: - logging.error('write_all_to_sock:unknown socket') - return True - - def _create_remote_socket(self, ip, port): - addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) - if len(addrs) == 0: - raise Exception("getaddrinfo failed for %s:%d" % (ip, port)) - af, socktype, proto, canonname, sa = addrs[0] - if self._forbidden_iplist: - if common.to_str(sa[0]) in self._forbidden_iplist: - raise Exception('IP %s is in forbidden list, reject' % - common.to_str(sa[0])) - remote_sock = socket.socket(af, socktype, proto) - self._remote_sock = remote_sock - - self._fd_to_handlers[remote_sock.fileno()] = self - - remote_sock.setblocking(False) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - return remote_sock - - def _handle_dns_resolved(self, result, error): - if error: - self._log_error(error) - self.destroy() - return - if result: - ip = result[1] - if ip: - - try: - self._stage = STAGE_CONNECTING - remote_addr = ip - remote_port = self._remote_address[1] - logging.info("connect to %s : %d" % (remote_addr, remote_port)) - - remote_sock = self._create_remote_socket(remote_addr, - remote_port) - try: - remote_sock.connect((remote_addr, remote_port)) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) in (errno.EINPROGRESS, - errno.EWOULDBLOCK): - pass # always goto here - else: - raise e - - self._loop.add(remote_sock, - eventloop.POLL_ERR | eventloop.POLL_OUT, - self._server) - self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) - self._stage = STAGE_STREAM - - addr = self.get_local_address() - - for i in range(2): - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) - self._write_to_sock(rsp_data, self._local_sock, addr) - - return - except Exception as e: - shell.print_exception(e) - if self._config['verbose']: - traceback.print_exc() - self.destroy() - - def _on_local_read(self): - # handle all local read events and dispatch them to methods for - # each stage - self._update_activity() - if not self._local_sock: - return - data = None - try: - data = self._local_sock.recv(BUF_SIZE) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) in \ - (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): - return - if not data: - self.destroy() - return - if not data: - return - self._server.server_transfer_ul += len(data) - #TODO ============================================================ - if self._stage == STAGE_STREAM: - self._write_to_sock(data, self._remote_sock) - return - - def _on_remote_read(self): - # handle all remote read events - self._update_activity() - data = None - try: - data = self._remote_sock.recv(BUF_SIZE) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) in \ - (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK, 10035): #errno.WSAEWOULDBLOCK - return - if not data: - self.destroy() - return - try: - self._server.server_transfer_dl += len(data) - recv_data = data - beg_pos = 0 - max_len = len(recv_data) - while beg_pos < max_len: - if beg_pos + POST_MTU_MAX >= max_len: - split_pos = max_len - else: - split_pos = beg_pos + self._random_mtu_size[self._random_mtu_index] - self._random_mtu_index = (self._random_mtu_index + 1) & 0x3ff - #split_pos = beg_pos + random.randint(POST_MTU_MIN, POST_MTU_MAX) - data = recv_data[beg_pos:split_pos] - beg_pos = split_pos - - pack_id = self._sendingqueue.append(data) - post_data = self._pack_post_data(CMD_POST, pack_id, data) - addr = self.get_local_address() - self._write_to_sock(post_data, self._local_sock, addr) - if pack_id <= DOUBLE_SEND_BEG_IDS: - post_data = self._pack_post_data(CMD_POST, pack_id, data) - self._write_to_sock(post_data, self._local_sock, addr) - - except Exception as e: - shell.print_exception(e) - if self._config['verbose']: - traceback.print_exc() - # TODO use logging when debug completed - self.destroy() - - def _on_local_write(self): - # handle local writable event - if self._data_to_write_to_local: - data = b''.join(self._data_to_write_to_local) - self._data_to_write_to_local = [] - self._write_to_sock(data, self._local_sock) - else: - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) - - def _on_remote_write(self): - # handle remote writable event - self._stage = STAGE_STREAM - if self._data_to_write_to_remote: - data = b''.join(self._data_to_write_to_remote) - self._data_to_write_to_remote = [] - self._write_to_sock(data, self._remote_sock) - else: - self._update_stream(STREAM_UP, WAIT_STATUS_READING) - - def _on_local_error(self): - logging.debug('got local error') - if self._local_sock: - logging.error(eventloop.get_sock_error(self._local_sock)) - self.destroy() - - def _on_remote_error(self): - logging.debug('got remote error') - if self._remote_sock: - logging.error(eventloop.get_sock_error(self._remote_sock)) - self.destroy() - - def _pack_rsp_data(self, cmd, data): - reqid_str = struct.pack(">H", self._request_id) - return b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, data, self._rand_data[:random.randint(0, len(self._rand_data))], reqid_str]) - - def _pack_rnd_data(self, data): - length = random.randint(0, len(self._rand_data)) - if length == 0: - return data - elif length == 1: - return b"\x81" + data - elif length < 256: - return b"\x80" + common.chr(length) + self._rand_data[:length - 2] + data - else: - return b"\x82" + struct.pack(">H", length) + self._rand_data[:length - 3] + data - - def _pack_post_data(self, cmd, pack_id, data): - reqid_str = struct.pack(">H", self._request_id) - recv_id = self._recvqueue.get_begin_id() - rsp_data = b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, struct.pack(">I", recv_id), struct.pack(">I", pack_id), data, reqid_str]) - return rsp_data - - def _pack_post_data_64(self, cmd, send_id, pack_id, data): - reqid_str = struct.pack(">H", self._request_id) - recv_id = self._recvqueue.get_begin_id() - rsp_data = b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, struct.pack(">Q", recv_id), struct.pack(">Q", pack_id), data, reqid_str]) - return rsp_data - - def sweep_timeout(self): - logging.info("sweep_timeout") - if self._stage == STAGE_STREAM: - pack_id, missing = self._recvqueue.get_missing_id(0) - logging.info("sweep_timeout %s %s" % (pack_id, missing)) - data = b'' - for pid in missing: - data += struct.pack(">H", pid) - rsp_data = self._pack_post_data(CMD_SYN_STATUS, pack_id, data) - addr = self.get_local_address() - self._write_to_sock(rsp_data, self._local_sock, addr) - - def handle_stream_sync_status(self, addr, cmd, request_id, pack_id, max_send_id, data): - missing_list = [] - while len(data) >= 2: - pid = struct.unpack(">H", data[0:2])[0] - data = data[2:] - missing_list.append(pid) - done_list = [] - self._recvqueue.set_end(max_send_id) - self._sendingqueue.set_finish(pack_id, done_list) - - if self._stage == STAGE_DESTROYED and self._sendingqueue.empty(): - self.destroy_local() - return - - # post CMD_SYN_STATUS - send_id = self._sendingqueue.get_end_id() - post_pack_id, missing = self._recvqueue.get_missing_id(0) - pack_ids_data = b'' - for pid in missing: - pack_ids_data += struct.pack(">H", pid) - - rsp_data = self._pack_rnd_data(self._pack_post_data(CMD_SYN_STATUS, send_id, pack_ids_data)) - self._write_to_sock(rsp_data, self._local_sock, addr) - - send_list = self._sendingqueue.get_data_list(pack_id, missing_list) - for post_pack_id, post_data in send_list: - rsp_data = self._pack_post_data(CMD_POST, post_pack_id, post_data) - self._write_to_sock(rsp_data, self._local_sock, addr) - if post_pack_id <= DOUBLE_SEND_BEG_IDS: - rsp_data = self._pack_post_data(CMD_POST, post_pack_id, post_data) - self._write_to_sock(rsp_data, self._local_sock, addr) - - def handle_client(self, addr, cmd, request_id, data): - self.add_local_address(addr) - if cmd == CMD_DISCONNECT: - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - self._write_to_sock(rsp_data, self._local_sock, addr) - self.destroy() - self.destroy_local() - return - if self._stage == STAGE_INIT: - if cmd == CMD_CONNECT: - self._request_id = request_id - self._stage = STAGE_RSP_ID - return - if self._request_id != request_id: - return - - if self._stage == STAGE_RSP_ID: - if cmd == CMD_CONNECT: - for i in range(2): - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, RSP_STATE_CONNECTED) - self._write_to_sock(rsp_data, self._local_sock, addr) - elif cmd == CMD_CONNECT_REMOTE: - local_id = data[0:4] - if self._local_id == local_id: - data = data[4:] - header_result = parse_header(data) - if header_result is None: - return - connecttype, remote_addr, remote_port, header_length = header_result - self._remote_address = (common.to_str(remote_addr), remote_port) - self._stage = STAGE_DNS - self._dns_resolver.resolve(remote_addr, - self._handle_dns_resolved) - common.connect_log('TCPonUDP connect %s:%d from %s:%d' % (remote_addr, remote_port, addr[0], addr[1])) - else: - # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - self._write_to_sock(rsp_data, self._local_sock, addr) - elif self._stage == STAGE_CONNECTING: - if cmd == CMD_CONNECT_REMOTE: - local_id = data[0:4] - if self._local_id == local_id: - for i in range(2): - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) - self._write_to_sock(rsp_data, self._local_sock, addr) - else: - # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - self._write_to_sock(rsp_data, self._local_sock, addr) - elif self._stage == STAGE_STREAM: - if len(data) < 4: - # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - self._write_to_sock(rsp_data, self._local_sock, addr) - return - local_id = data[0:4] - if self._local_id != local_id: - # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - self._write_to_sock(rsp_data, self._local_sock, addr) - return - else: - data = data[4:] - if cmd == CMD_CONNECT_REMOTE: - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT_REMOTE, RSP_STATE_CONNECTEDREMOTE) - self._write_to_sock(rsp_data, self._local_sock, addr) - elif cmd == CMD_POST: - recv_id = struct.unpack(">I", data[0:4])[0] - pack_id = struct.unpack(">I", data[4:8])[0] - self._recvqueue.insert(pack_id, data[8:]) - self._sendingqueue.set_finish(recv_id, []) - elif cmd == CMD_POST_64: - recv_id = struct.unpack(">Q", data[0:8])[0] - pack_id = struct.unpack(">Q", data[8:16])[0] - self._recvqueue.insert(pack_id, data[16:]) - self._sendingqueue.set_finish(recv_id, []) - elif cmd == CMD_DISCONNECT: - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - self._write_to_sock(rsp_data, self._local_sock, addr) - self.destroy() - self.destroy_local() - return - elif cmd == CMD_SYN_STATUS: - pack_id = struct.unpack(">I", data[0:4])[0] - max_send_id = struct.unpack(">I", data[4:8])[0] - data = data[8:] - self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) - elif cmd == CMD_SYN_STATUS_64: - pack_id = struct.unpack(">Q", data[0:8])[0] - max_send_id = struct.unpack(">Q", data[8:16])[0] - data = data[16:] - self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) - while self._recvqueue.has_data(): - pack_id, post_data = self._recvqueue.get_data() - self._write_to_sock(post_data, self._remote_sock) - elif self._stage == STAGE_DESTROYED: - local_id = data[0:4] - if self._local_id != local_id: - # ileagal request - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - self._write_to_sock(rsp_data, self._local_sock, addr) - return - else: - data = data[4:] - if cmd == CMD_SYN_STATUS: - pack_id = struct.unpack(">I", data[0:4])[0] - max_send_id = struct.unpack(">I", data[4:8])[0] - data = data[8:] - self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) - elif cmd == CMD_SYN_STATUS_64: - pack_id = struct.unpack(">Q", data[0:8])[0] - max_send_id = struct.unpack(">Q", data[8:16])[0] - data = data[16:] - self.handle_stream_sync_status(addr, cmd, request_id, pack_id, max_send_id, data) - - def handle_event(self, sock, event): - # handle all events in this handler and dispatch them to methods - handle = False - if self._stage == STAGE_DESTROYED: - logging.debug('ignore handle_event: destroyed') - return True - # order is important - if sock == self._remote_sock: - if event & eventloop.POLL_ERR: - handle = True - self._on_remote_error() - if self._stage == STAGE_DESTROYED: - return True - if event & (eventloop.POLL_IN | eventloop.POLL_HUP): - handle = True - self._on_remote_read() - if self._stage == STAGE_DESTROYED: - return True - if event & eventloop.POLL_OUT: - handle = True - self._on_remote_write() - elif sock == self._local_sock: - if event & eventloop.POLL_ERR: - handle = True - self._on_local_error() - if self._stage == STAGE_DESTROYED: - return True - if event & (eventloop.POLL_IN | eventloop.POLL_HUP): - handle = True - self._on_local_read() - if self._stage == STAGE_DESTROYED: - return True - if event & eventloop.POLL_OUT: - handle = True - self._on_local_write() - else: - logging.warn('unknown socket') - - return handle - - def _log_error(self, e): - logging.error('%s when handling connection from %s' % - (e, self._client_address.keys())) - - def destroy(self): - # destroy the handler and release any resources - # promises: - # 1. destroy won't make another destroy() call inside - # 2. destroy releases resources so it prevents future call to destroy - # 3. destroy won't raise any exceptions - # if any of the promises are broken, it indicates a bug has been - # introduced! mostly likely memory leaks, etc - #logging.info('tcp destroy called') - if self._stage == STAGE_DESTROYED: - # this couldn't happen - logging.debug('already destroyed') - return - self._stage = STAGE_DESTROYED - if self._remote_address: - logging.debug('destroy: %s:%d' % - self._remote_address) - else: - logging.debug('destroy') - if self._remote_sock: - logging.debug('destroying remote') - self._loop.remove(self._remote_sock) - try: - del self._fd_to_handlers[self._remote_sock.fileno()] - except Exception as e: - pass - self._remote_sock.close() - self._remote_sock = None - if self._sendingqueue.empty(): - self.destroy_local() - self._dns_resolver.remove_callback(self._handle_dns_resolved) - - def destroy_local(self): - if self._local_sock: - logging.debug('disconnect local') - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, RSP_STATE_EMPTY) - addr = None - addr = self.get_local_address() - self._write_to_sock(rsp_data, self._local_sock, addr) - self._local_sock = None - try: - del self._reqid_to_handlers[self._request_id] - except Exception as e: - pass - - self._server.remove_handler(self) - def client_key(source_addr, server_af): # notice this is server af, not dest af return '%s:%s:%d' % (source_addr[0], source_addr[1], server_af) @@ -908,7 +163,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self.server_user_transfer_ul = {} self.server_user_transfer_dl = {} - if common.to_bytes(config['protocol']) in [b"auth_aes128_md5", b"auth_aes128_sha1"]: + if common.to_bytes(config['protocol']) in obfs.mu_protocol(): self._update_users(None, None) self.protocol_data = obfs.obfs(config['protocol']).init_data() @@ -1261,71 +516,6 @@ def _handle_server(self): else: shell.print_exception(e) - def _handle_tcp_over_udp(self, data, r_addr): - #(cmd, request_id, data) - #logging.info("UDP data %d %d %s" % (data[0], data[1], binascii.hexlify(data[2]))) - try: - self.server_transfer_ul += len(data[2]) - if data[0] == 0: - if len(data[2]) >= 4: - for i in range(64): - req_id = random.randint(1, 65535) - if req_id not in self._reqid_to_hd: - break - if req_id in self._reqid_to_hd: - for i in range(64): - req_id = random.randint(1, 65535) - if type(self._reqid_to_hd[req_id]) is tuple: - break - # return req id - self._reqid_to_hd[req_id] = (data[2][0:4], None) - rsp_data = self._pack_rsp_data(CMD_RSP_CONNECT, req_id, RSP_STATE_CONNECTED) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - elif data[0] == CMD_CONNECT_REMOTE: - if len(data[2]) > 4 and data[1] in self._reqid_to_hd: - # create - if type(self._reqid_to_hd[data[1]]) is tuple: - if data[2][0:4] == self._reqid_to_hd[data[1]][0]: - handle = TCPRelayHandler(self, self._reqid_to_hd, self._fd_to_handlers, - self._eventloop, self._server_socket, - self._reqid_to_hd[data[1]][0], self._reqid_to_hd[data[1]][1], - self._config, self._dns_resolver, self._is_local) - self._reqid_to_hd[data[1]] = handle - handle.handle_client(r_addr, CMD_CONNECT, data[1], data[2]) - handle.handle_client(r_addr, *data) - self.update_activity(handle) - else: - # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - else: - self.update_activity(self._reqid_to_hd[data[1]]) - self._reqid_to_hd[data[1]].handle_client(r_addr, *data) - else: - # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - elif data[0] > CMD_CONNECT_REMOTE and data[0] <= CMD_DISCONNECT: - if data[1] in self._reqid_to_hd: - if type(self._reqid_to_hd[data[1]]) is tuple: - pass - else: - self.update_activity(self._reqid_to_hd[data[1]]) - self._reqid_to_hd[data[1]].handle_client(r_addr, *data) - else: - # disconnect - rsp_data = self._pack_rsp_data(CMD_DISCONNECT, data[1], RSP_STATE_EMPTY) - data_to_send = encrypt.encrypt_all(self._password, self._method, 1, rsp_data) - self.write_to_server_socket(data_to_send, r_addr) - return - except Exception as e: - trace = traceback.format_exc() - logging.error(trace) - return - def _handle_client(self, sock): data, r_addr = sock.recvfrom(BUF_SIZE) if not data: @@ -1333,6 +523,18 @@ def _handle_client(self, sock): return if self._stat_callback: self._stat_callback(self._listen_port, len(data)) + + client_addr = self._client_fd_to_server_addr.get(sock.fileno()) + client_uid = None + if client_addr: + key = client_key(client_addr[0], client_addr[1]) + client_pair = self._cache.get(key, None) + client_dns_pair = self._cache_dns_client.get(key, None) + if client_pair: + client, client_uid = client_pair + elif client_dns_pair: + client, client_uid = client_dns_pair + if not self._is_local: addrlen = len(r_addr[0]) if addrlen > 255: @@ -1341,7 +543,7 @@ def _handle_client(self, sock): data = pack_addr(r_addr[0]) + struct.pack('>H', r_addr[1]) + data ref_iv = [encrypt.encrypt_new_iv(self._method)] self._protocol.obfs.server_info.iv = ref_iv[0] - data = self._protocol.server_udp_pre_encrypt(data) + data = self._protocol.server_udp_pre_encrypt(data, client_uid) response = encrypt.encrypt_all_iv(self._protocol.obfs.server_info.key, self._method, 1, data, ref_iv) if not response: @@ -1361,16 +563,9 @@ def _handle_client(self, sock): #logging.debug('UDP handle_client %s:%d to %s:%d' % (common.to_str(r_addr[0]), r_addr[1], dest_addr, dest_port)) response = b'\x00\x00\x00' + data - client_addr = self._client_fd_to_server_addr.get(sock.fileno()) + if client_addr: - key = client_key(client_addr[0], client_addr[1]) - client_pair = self._cache.get(key, None) - client_dns_pair = self._cache_dns_client.get(key, None) - if client_pair: - client, client_uid = client_pair - self.add_transfer_d(client_uid, len(response)) - elif client_dns_pair: - client, client_uid = client_dns_pair + if client_uid: self.add_transfer_d(client_uid, len(response)) else: self.server_transfer_dl += len(response) From caadb606ba42cac919367ddd036376a44ec6ec31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 25 Apr 2017 16:38:04 +0800 Subject: [PATCH 405/487] impl auth_chain client --- shadowsocks/obfsplugin/auth_chain.py | 71 +++++++++++++++------------- shadowsocks/tcprelay.py | 57 ++++++++++++---------- shadowsocks/udprelay.py | 1 + 3 files changed, 72 insertions(+), 57 deletions(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index ff0a812f..c23a883b 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -336,34 +336,35 @@ def pack_server_data(self, buf): return data def pack_auth_data(self, auth_data, buf): - if len(buf) == 0: - return b'' - if len(buf) > 400: - rnd_len = struct.unpack(' 4: mac_key = self.user_key + struct.pack('= 8192 or length < 7: + data_len = struct.unpack('= 4096: self.raw_trans = True self.recv_buf = b'' raise Exception('client_post_decrypt data error') - if length > len(self.recv_buf): + + if length + 4 > len(self.recv_buf): break - if hmac.new(mac_key, self.recv_buf[:length - 4], self.hashfunc).digest()[:4] != self.recv_buf[length - 4:length]: + server_hash = hmac.new(mac_key, self.recv_buf[:length + 2], self.hashfunc).digest() + if server_hash[:2] != self.recv_buf[length + 2 : length + 4]: + logging.info('%s: checksum error, data %s' % (self.no_compatible_method, binascii.hexlify(self.recv_buf[:length]))) self.raw_trans = True self.recv_buf = b'' raise Exception('client_post_decrypt data uncorrect checksum') + pos = 2 + if data_len > 0 and rand_len > 0: + pos = 2 + self.rnd_start_pos(rand_len, self.random_server) + out_buf += self.encryptor.decrypt(self.recv_buf[pos : data_len + pos]) + self.last_server_hash = server_hash + if self.recv_id == 1: + self.server_info.tcp_mss = out_buf[:2] + out_buf = out_buf[2:] self.recv_id = (self.recv_id + 1) & 0xFFFFFFFF - pos = common.ord(self.recv_buf[4]) - if pos < 255: - pos += 4 - else: - pos = struct.unpack(' Date: Tue, 25 Apr 2017 17:00:08 +0800 Subject: [PATCH 406/487] fix auth_chain_a --- shadowsocks/obfsplugin/auth_chain.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index c23a883b..7cf1d73f 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -313,6 +313,7 @@ def rnd_data(self, buf_size, buf, last_hash, random): return buf def pack_client_data(self, buf): + buf = self.encryptor.encrypt(buf) data = self.rnd_data(len(buf), buf, self.last_client_hash, self.random_client) data_len = len(data) + 8 mac_key = self.user_key + struct.pack(' Date: Thu, 27 Apr 2017 18:17:35 +0800 Subject: [PATCH 407/487] bump --- CHANGES | 4 ++++ shadowsocks/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 00108900..072bb1d7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +3.2.0 2017-04-27 +- add auth_chain_a +- remove auth_aes128, auth_sha1, auth_sha1_v2, verify_simple, auth_simple, verify_sha1 + 3.1.2 2017-04-07 - display UID - auto adjust TCP MSS diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 882c72e0..eea9c31d 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.1.2 2017-04-07' + return '3.2.0 2017-04-27' From 582d7b72564e8b38f276309951ff19069ca5214d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 27 Apr 2017 21:28:23 +0800 Subject: [PATCH 408/487] connect_log include local addr & port --- shadowsocks/tcprelay.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index dbc428ab..2687a73e 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -596,9 +596,12 @@ def _handle_stage_addr(self, ogn_data, data): server_info = self._protocol.get_server_info() server_info.buffer_size = self._recv_buffer_size connecttype, remote_addr, remote_port, header_length = header_result - common.connect_log('%s connecting %s:%d via port %d by UID %d' % - ((connecttype == 0) and 'TCP' or 'UDP', - common.to_str(remote_addr), remote_port, self._server._listen_port, self._user_id)) + if connecttype != 0: + common.connect_log('UDP over TCP by user %d' % + (self._user_id, )) + else: + common.connect_log('TCP request %s:%d by user %d' % + (common.to_str(remote_addr), remote_port, self._user_id)) self._remote_address = (common.to_str(remote_addr), remote_port) self._remote_udp = (connecttype != 0) # pause reading @@ -746,6 +749,10 @@ def _handle_dns_resolved(self, result, error): pass # always goto here else: raise e + addr, port = self._remote_sock.getsockname()[:2] + common.connect_log('TCP connecting %s(%s):%d from %s:%d by user %d' % + (self._remote_address[0], remote_addr, remote_port, addr, port, self._user_id)) + self._loop.add(remote_sock, eventloop.POLL_ERR | eventloop.POLL_OUT, self._server) From 5896eac1a8808338fa551d44d9223b42504011e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 28 Apr 2017 21:43:18 +0800 Subject: [PATCH 409/487] fix auth_chain_a UDP bug --- shadowsocks/obfsplugin/auth_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index 7cf1d73f..5eed68cd 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -602,7 +602,7 @@ def server_udp_pre_encrypt(self, buf, uid): mac_key = self.server_info.key md5data = hmac.new(mac_key, authdata, self.hashfunc).digest() rand_len = self.udp_rnd_data_len(md5data, self.random_server) - encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(self.user_key)) + to_bytes(base64.b64encode(md5data)), 'rc4') + encryptor = encrypt.Encryptor(to_bytes(base64.b64encode(user_key)) + to_bytes(base64.b64encode(md5data)), 'rc4') out_buf = encryptor.encrypt(buf) buf = out_buf + os.urandom(rand_len) + authdata return buf + hmac.new(user_key, buf, self.hashfunc).digest()[:1] From f3a3ab4b84ecc33f719a87427dff631e01a00380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 28 Apr 2017 21:46:47 +0800 Subject: [PATCH 410/487] log UDP bind port --- shadowsocks/tcprelay.py | 19 +++++++++++++++---- shadowsocks/udprelay.py | 11 ++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 2687a73e..c26109fa 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -223,6 +223,8 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self.speed_tester_u = SpeedTester(config.get("speed_limit_per_con", 0)) self.speed_tester_d = SpeedTester(config.get("speed_limit_per_con", 0)) self._recv_pack_id = 0 + self._udp_send_pack_id = 0 + self._udpv6_send_pack_id = 0 def __hash__(self): # default __hash__ is id / 16 @@ -338,14 +340,23 @@ def _write_to_sock(self, data, sock): connecttype, dest_addr, dest_port, header_length = header_result addrs = socket.getaddrinfo(dest_addr, dest_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) - #logging.info('UDP over TCP sendto %s:%d %d bytes from %s:%d' % (dest_addr, dest_port, len(data), self._client_address[0], self._client_address[1])) if addrs: af, socktype, proto, canonname, server_addr = addrs[0] data = data[header_length:] if af == socket.AF_INET6: self._remote_sock_v6.sendto(data, (server_addr[0], dest_port)) + if self._udpv6_send_pack_id == 0: + addr, port = self._remote_sock_v6.getsockname()[:2] + common.connect_log('UDPv6 sendto %s:%d from %s:%d by user %d' % + (server_addr[0], dest_port, addr, port, self._user_id)) + self._udpv6_send_pack_id += 1 else: sock.sendto(data, (server_addr[0], dest_port)) + if self._udp_send_pack_id == 0: + addr, port = sock.getsockname()[:2] + common.connect_log('UDP sendto %s:%d from %s:%d by user %d' % + (server_addr[0], dest_port, addr, port, self._user_id)) + self._udp_send_pack_id += 1 except Exception as e: #trace = traceback.format_exc() @@ -597,8 +608,9 @@ def _handle_stage_addr(self, ogn_data, data): server_info.buffer_size = self._recv_buffer_size connecttype, remote_addr, remote_port, header_length = header_result if connecttype != 0: - common.connect_log('UDP over TCP by user %d' % - (self._user_id, )) + pass + #common.connect_log('UDP over TCP by user %d' % + # (self._user_id, )) else: common.connect_log('TCP request %s:%d by user %d' % (common.to_str(remote_addr), remote_port, self._user_id)) @@ -711,7 +723,6 @@ def _handle_dns_resolved(self, result, error): if result: ip = result[1] if ip: - try: self._stage = STAGE_CONNECTING remote_addr = ip diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 05ecee30..fa9f404d 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -445,9 +445,9 @@ def _handle_server(self): af, socktype, proto, canonname, sa = addrs[0] key = client_key(r_addr, af) client_pair = self._cache.get(key, None) - if not client_pair: + if client_pair is None: client_pair = self._cache_dns_client.get(key, None) - if not client_pair: + if client_pair is None: if self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: logging.debug('IP %s is in forbidden list, drop' % @@ -486,9 +486,6 @@ def _handle_server(self): user_id = self._listen_port else: user_id = struct.unpack(' Date: Sat, 29 Apr 2017 02:46:25 +0800 Subject: [PATCH 411/487] force tcp_mss in range (500, 1500] --- shadowsocks/obfsplugin/auth.py | 2 +- shadowsocks/tcprelay.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/shadowsocks/obfsplugin/auth.py b/shadowsocks/obfsplugin/auth.py index 51a1b00c..a745e098 100755 --- a/shadowsocks/obfsplugin/auth.py +++ b/shadowsocks/obfsplugin/auth.py @@ -501,7 +501,7 @@ def trapezoid_random_int(self, max_val, d): def rnd_data_len(self, buf_size, full_buf_size): if full_buf_size >= self.server_info.buffer_size: return 0 - tcp_mss = self.server_info.tcp_mss if self.server_info.tcp_mss < 1500 else 1500 + tcp_mss = self.server_info.tcp_mss rev_len = tcp_mss - buf_size - 9 if rev_len == 0: return 0 diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c26109fa..3096e00b 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -248,7 +248,9 @@ def _get_a_server(self): def _update_tcp_mss(self, local_sock): self._tcp_mss = TCP_MSS try: - self._tcp_mss = local_sock.getsockopt(socket.SOL_TCP, socket.TCP_MAXSEG) + tcp_mss = local_sock.getsockopt(socket.SOL_TCP, socket.TCP_MAXSEG) + if tcp_mss > 500 and tcp_mss <= 1500: + self._tcp_mss = tcp_mss logging.debug("TCP MSS = %d" % (self._tcp_mss,)) except: pass From 3c308ca876e3235a4cfdad5b07d42c71f297a490 Mon Sep 17 00:00:00 2001 From: LI JIAHAO Date: Sat, 29 Apr 2017 08:32:35 +0800 Subject: [PATCH 412/487] force additional ports only with ss-panel (#218) --- config.json | 1 + db_transfer.py | 5 +++-- shadowsocks/shell.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index 988100f7..55f12b5f 100644 --- a/config.json +++ b/config.json @@ -15,6 +15,7 @@ "speed_limit_per_user": 0, "additional_ports" : {}, // only works under multi-user mode + "additional_ports_only" : false, // only works under multi-user mode "timeout": 120, "udp_timeout": 60, "dns_ipv6": false, diff --git a/db_transfer.py b/db_transfer.py index a5fb8623..17a28fca 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -152,8 +152,9 @@ def del_server_out_of_bound_safe(self, last_rows, rows): else: self.new_server(port, passwd, cfg) else: + config = shell.get_config(False) if ServerPool.get_instance().server_is_run(port) > 0: - if not allow: + if config['additional_ports_only'] or not allow: logging.info('db stop server at port [%s]' % (port,)) ServerPool.get_instance().cb_del_server(port) self.force_update_transfer.add(port) @@ -164,7 +165,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): self.force_update_transfer.add(port) new_servers[port] = (passwd, cfg) - elif allow and port > 0 and port < 65536 and ServerPool.get_instance().server_run_status(port) is False: + elif not config['additional_ports_only'] and allow and port > 0 and port < 65536 and ServerPool.get_instance().server_run_status(port) is False: self.new_server(port, passwd, cfg) for row in last_rows: diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index 1540412a..f0e0cb01 100755 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -245,6 +245,7 @@ def get_config(is_local): config['obfs_param'] = to_str(config.get('obfs_param', '')) config['port_password'] = config.get('port_password', None) config['additional_ports'] = config.get('additional_ports', {}) + config['additional_ports_only'] = config.get('additional_ports_only', False) config['timeout'] = int(config.get('timeout', 300)) config['udp_timeout'] = int(config.get('udp_timeout', 120)) config['udp_cache'] = int(config.get('udp_cache', 64)) From c3bb64bc2ca56ab54f4eb82a51e88dd9793b9d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 30 Apr 2017 08:35:32 +0800 Subject: [PATCH 413/487] add interface legendsockssr --- apiconfig.py | 2 +- db_transfer.py | 111 ++++++++++++++++++++++++++----------------------- 2 files changed, 61 insertions(+), 52 deletions(-) diff --git a/apiconfig.py b/apiconfig.py index cbeba7f0..5ee8be12 100644 --- a/apiconfig.py +++ b/apiconfig.py @@ -1,5 +1,5 @@ # Config -API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, glzjinmod, muapiv2(not support) +API_INTERFACE = 'sspanelv2' #mudbjson, sspanelv2, sspanelv3, sspanelv3ssr, glzjinmod, legendsockssr, muapiv2(not support) UPDATE_TIME = 60 SERVER_PUB_ADDR = '127.0.0.1' # mujson_mgr need this to generate ssr link diff --git a/db_transfer.py b/db_transfer.py index 17a28fca..4bb7ba6f 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -410,13 +410,19 @@ def pull_db_users(self, conn): class Dbv3Transfer(DbTransfer): def __init__(self): super(Dbv3Transfer, self).__init__() - self.key_list += ['id', 'method'] - self.ss_node_info_name = 'ss_node_info_log' - if get_config().API_INTERFACE == 'sspanelv3ssr': - self.key_list += ['obfs', 'protocol'] - if get_config().API_INTERFACE == 'glzjinmod': + self.update_node_state = True if get_config().API_INTERFACE != 'legendsockssr' else False + if self.update_node_state: + self.key_list += ['id'] + self.key_list += ['method'] + if self.update_node_state: + self.ss_node_info_name = 'ss_node_info_log' + if get_config().API_INTERFACE == 'sspanelv3ssr': + self.key_list += ['obfs', 'protocol'] + if get_config().API_INTERFACE == 'glzjinmod': + self.key_list += ['obfs', 'protocol'] + self.ss_node_info_name = 'ss_node_info' + else: self.key_list += ['obfs', 'protocol'] - self.ss_node_info_name = 'ss_node_info' self.start_time = time.time() def update_all_user(self, dt_transfer): @@ -458,16 +464,17 @@ def update_all_user(self, dt_transfer): query_sub_when2 += ' WHEN %s THEN d+%s' % (id, int(transfer[1] * self.cfg["transfer_mul"])) update_transfer[id] = transfer - cur = conn.cursor() - try: - if id in self.port_uid_table: - cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `node_id`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ - str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ - str(self.cfg["node_id"]) + "', '" + str(self.cfg["transfer_mul"]) + "', '" + \ - self.traffic_format((transfer[0] + transfer[1]) * self.cfg["transfer_mul"]) + "', unix_timestamp()); ") - except: - logging.warn('no `user_traffic_log` in db') - cur.close() + if self.update_node_state: + cur = conn.cursor() + try: + if id in self.port_uid_table: + cur.execute("INSERT INTO `user_traffic_log` (`id`, `user_id`, `u`, `d`, `node_id`, `rate`, `traffic`, `log_time`) VALUES (NULL, '" + \ + str(self.port_uid_table[id]) + "', '" + str(transfer[0]) + "', '" + str(transfer[1]) + "', '" + \ + str(self.cfg["node_id"]) + "', '" + str(self.cfg["transfer_mul"]) + "', '" + \ + self.traffic_format((transfer[0] + transfer[1]) * self.cfg["transfer_mul"]) + "', unix_timestamp()); ") + except: + logging.warn('no `user_traffic_log` in db') + cur.close() if query_sub_in is not None: query_sub_in += ',%s' % id @@ -486,25 +493,26 @@ def update_all_user(self, dt_transfer): logging.error(e) cur.close() - try: - cur = conn.cursor() + if self.update_node_state: try: - cur.execute("INSERT INTO `ss_node_online_log` (`id`, `node_id`, `online_user`, `log_time`) VALUES (NULL, '" + \ - str(self.cfg["node_id"]) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") - except Exception as e: - logging.error(e) - cur.close() + cur = conn.cursor() + try: + cur.execute("INSERT INTO `ss_node_online_log` (`id`, `node_id`, `online_user`, `log_time`) VALUES (NULL, '" + \ + str(self.cfg["node_id"]) + "', '" + str(alive_user_count) + "', unix_timestamp()); ") + except Exception as e: + logging.error(e) + cur.close() - cur = conn.cursor() - try: - cur.execute("INSERT INTO `" + self.ss_node_info_name + "` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ - str(self.cfg["node_id"]) + "', '" + str(self.uptime()) + "', '" + \ - str(self.load()) + "', unix_timestamp()); ") - except Exception as e: - logging.error(e) - cur.close() - except: - logging.warn('no `ss_node_online_log` or `" + self.ss_node_info_name + "` in db') + cur = conn.cursor() + try: + cur.execute("INSERT INTO `" + self.ss_node_info_name + "` (`id`, `node_id`, `uptime`, `load`, `log_time`) VALUES (NULL, '" + \ + str(self.cfg["node_id"]) + "', '" + str(self.uptime()) + "', '" + \ + str(self.load()) + "', unix_timestamp()); ") + except Exception as e: + logging.error(e) + cur.close() + except: + logging.warn('no `ss_node_online_log` or `" + self.ss_node_info_name + "` in db') conn.close() return update_transfer @@ -518,26 +526,27 @@ def pull_db_users(self, conn): cur = conn.cursor() - node_info_keys = ['traffic_rate'] - try: - cur.execute("SELECT " + ','.join(node_info_keys) +" FROM ss_node where `id`='" + str(self.cfg["node_id"]) + "'") - nodeinfo = cur.fetchone() - except Exception as e: - logging.error(e) - nodeinfo = None - - if nodeinfo == None: - rows = [] + if self.update_node_state: + node_info_keys = ['traffic_rate'] + try: + cur.execute("SELECT " + ','.join(node_info_keys) +" FROM ss_node where `id`='" + str(self.cfg["node_id"]) + "'") + nodeinfo = cur.fetchone() + except Exception as e: + logging.error(e) + nodeinfo = None + + if nodeinfo == None: + rows = [] + cur.close() + conn.commit() + logging.warn('None result when select node info from ss_node in db, maybe you set the incorrect node id') + return rows cur.close() - conn.commit() - logging.warn('None result when select node info from ss_node in db, maybe you set the incorrect node id') - return rows - cur.close() - node_info_dict = {} - for column in range(len(nodeinfo)): - node_info_dict[node_info_keys[column]] = nodeinfo[column] - self.cfg['transfer_mul'] = float(node_info_dict['traffic_rate']) + node_info_dict = {} + for column in range(len(nodeinfo)): + node_info_dict[node_info_keys[column]] = nodeinfo[column] + self.cfg['transfer_mul'] = float(node_info_dict['traffic_rate']) cur = conn.cursor() try: From 1e85d14a48d17eedc4afe699878142be955f7211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 30 Apr 2017 21:26:36 +0800 Subject: [PATCH 414/487] optimize --- db_transfer.py | 2 +- shadowsocks/udprelay.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 4bb7ba6f..0f3a7c95 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -88,6 +88,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): new_servers = {} allow_users = {} mu_servers = {} + config = shell.get_config(False) for row in rows: try: allow = switchrule.isTurnOn(row) and row['enable'] == 1 and row['u'] + row['d'] < row['transfer_enable'] @@ -152,7 +153,6 @@ def del_server_out_of_bound_safe(self, last_rows, rows): else: self.new_server(port, passwd, cfg) else: - config = shell.get_config(False) if ServerPool.get_instance().server_is_run(port) > 0: if config['additional_ports_only'] or not allow: logging.info('db stop server at port [%s]' % (port,)) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index fa9f404d..f1177817 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -506,11 +506,11 @@ def _handle_server(self): try: #logging.info('UDP handle_server sendto %s:%d %d bytes' % (common.to_str(server_addr), server_port, len(data))) client.sendto(data, (server_addr, server_port)) + self.add_transfer_u(client_uid, len(data)) if client_pair is None: # new request addr, port = client.getsockname()[:2] common.connect_log('UDP data to %s:%d from %s:%d by UID %d' % (common.to_str(server_addr), server_port, addr, port, user_id)) - self.add_transfer_u(client_uid, len(data)) except IOError as e: err = eventloop.errno_from_exception(e) if err in (errno.EINPROGRESS, errno.EAGAIN): From 03b1a53b9a238173057301ee964f04170d5e403e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 2 May 2017 12:40:23 +0800 Subject: [PATCH 415/487] run with newest python version --- logrun.sh | 5 +++-- run.sh | 5 +++-- shadowsocks/logrun.sh | 8 ++++--- shadowsocks/run.sh | 7 +++--- shadowsocks/stop.sh | 4 +++- shadowsocks/udprelay.py | 50 +++-------------------------------------- stop.sh | 4 +++- 7 files changed, 24 insertions(+), 59 deletions(-) diff --git a/logrun.sh b/logrun.sh index 82a2105f..0411f51b 100644 --- a/logrun.sh +++ b/logrun.sh @@ -1,6 +1,7 @@ #!/bin/bash cd `dirname $0` -eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') +python_ver=$(ls /usr/lib|grep "^python"|tail -1) +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') ulimit -n 512000 -nohup python server.py m>> ssserver.log 2>&1 & +nohup ${python_ver} server.py m>> ssserver.log 2>&1 & diff --git a/run.sh b/run.sh index 89f7b05b..16f1f64d 100644 --- a/run.sh +++ b/run.sh @@ -1,6 +1,7 @@ #!/bin/bash cd `dirname $0` -eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') +python_ver=$(ls /usr/lib|grep "^python"|tail -1) +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') ulimit -n 512000 -nohup python server.py m>> /dev/null 2>&1 & +nohup ${python_ver} server.py m>> /dev/null 2>&1 & diff --git a/shadowsocks/logrun.sh b/shadowsocks/logrun.sh index 2b7a5bd8..eb3afe27 100644 --- a/shadowsocks/logrun.sh +++ b/shadowsocks/logrun.sh @@ -1,6 +1,8 @@ #!/bin/bash cd `dirname $0` -eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') -ulimit -n 4096 -nohup python server.py a >> ssserver.log 2>&1 & +python_ver=$(ls /usr/lib|grep "^python"|tail -1) +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') +ulimit -n 512000 +nohup ${python_ver} server.py a>> ssserver.log 2>&1 & + diff --git a/shadowsocks/run.sh b/shadowsocks/run.sh index 7a91779c..7950c4c9 100644 --- a/shadowsocks/run.sh +++ b/shadowsocks/run.sh @@ -1,6 +1,7 @@ #!/bin/bash cd `dirname $0` -eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') -ulimit -n 4096 -nohup python server.py a >> /dev/null 2>&1 & +python_ver=$(ls /usr/lib|grep "^python"|tail -1) +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') +ulimit -n 512000 +nohup ${python_ver} server.py a>> /dev/null 2>&1 & diff --git a/shadowsocks/stop.sh b/shadowsocks/stop.sh index af1fbf92..81398909 100644 --- a/shadowsocks/stop.sh +++ b/shadowsocks/stop.sh @@ -1,3 +1,5 @@ #!/bin/bash -eval $(ps -ef | grep "[0-9] python server\\.py a" | awk '{print "kill "$2}') +python_ver=$(ls /usr/lib|grep "^python"|tail -1) +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') + diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index f1177817..c1539791 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -311,41 +311,6 @@ def _close_client(self, client): client.info('close_client pass %s' % client) pass - def _pre_parse_udp_header(self, data): - if data is None: - return - datatype = common.ord(data[0]) - if datatype == 0x8: - if len(data) >= 8: - crc = binascii.crc32(data) & 0xffffffff - if crc != 0xffffffff: - logging.warn('uncorrect CRC32, maybe wrong password or ' - 'encryption method') - return None - cmd = common.ord(data[1]) - request_id = struct.unpack('>H', data[2:4])[0] - data = data[4:-4] - return (cmd, request_id, data) - elif len(data) >= 6 and common.ord(data[1]) == 0x0: - crc = binascii.crc32(data) & 0xffffffff - if crc != 0xffffffff: - logging.warn('uncorrect CRC32, maybe wrong password or ' - 'encryption method') - return None - cmd = common.ord(data[1]) - data = data[2:-4] - return (cmd, 0, data) - else: - logging.warn('header too short, maybe wrong password or ' - 'encryption method') - return None - return data - - def _pack_rsp_data(self, cmd, request_id, data): - _rand_data = b"123456789abcdefghijklmnopqrstuvwxyz" * 2 - reqid_str = struct.pack(">H", request_id) - return b''.join([CMD_VER_STR, common.chr(cmd), reqid_str, data, _rand_data[:random.randint(0, len(_rand_data))], reqid_str]) - def _handel_protocol_error(self, client_address, ogn_data): #raise Exception('can not parse header') logging.warn("Protocol ERROR, UDP ogn data %s from %s:%d" % (binascii.hexlify(ogn_data), client_address[0], client_address[1])) @@ -398,15 +363,9 @@ def _handle_server(self): #logging.info("UDP data %s" % (binascii.hexlify(data),)) if not self._is_local: data = pre_parse_header(data) - - data = self._pre_parse_udp_header(data) if data is None: return - if type(data) is tuple: - return - #return self._handle_tcp_over_udp(data, r_addr) - try: header_result = parse_header(data) except: @@ -450,14 +409,12 @@ def _handle_server(self): if client_pair is None: if self._forbidden_iplist: if common.to_str(sa[0]) in self._forbidden_iplist: - logging.debug('IP %s is in forbidden list, drop' % - common.to_str(sa[0])) + logging.debug('IP %s is in forbidden list, drop' % common.to_str(sa[0])) # drop return if self._forbidden_portset: if sa[1] in self._forbidden_portset: - logging.debug('Port %d is in forbidden list, reject' % - sa[1]) + logging.debug('Port %d is in forbidden list, reject' % sa[1]) # drop return client = socket.socket(af, socktype, proto) @@ -469,7 +426,6 @@ def _handle_server(self): is_dns = True else: pass - #logging.info("unknown data %s" % (binascii.hexlify(data),)) if sa[1] == 53 and is_dns: #DNS logging.debug("DNS query %s from %s:%d" % (common.to_str(sa[0]), r_addr[0], r_addr[1])) self._cache_dns_client[key] = (client, uid) @@ -504,7 +460,6 @@ def _handle_server(self): if not data: return try: - #logging.info('UDP handle_server sendto %s:%d %d bytes' % (common.to_str(server_addr), server_port, len(data))) client.sendto(data, (server_addr, server_port)) self.add_transfer_u(client_uid, len(data)) if client_pair is None: # new request @@ -513,6 +468,7 @@ def _handle_server(self): (common.to_str(server_addr), server_port, addr, port, user_id)) except IOError as e: err = eventloop.errno_from_exception(e) + logging.warning('IOError sendto %s:%d by user %d' % (server_addr, server_port, user_id)) if err in (errno.EINPROGRESS, errno.EAGAIN): pass else: diff --git a/stop.sh b/stop.sh index c726221a..edca32f3 100644 --- a/stop.sh +++ b/stop.sh @@ -1,3 +1,5 @@ #!/bin/bash -eval $(ps -ef | grep "[0-9] python server\\.py m" | awk '{print "kill "$2}') +python_ver=$(ls /usr/lib|grep "^python"|tail -1) +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') + From d968f01245fbc527c2901040e4b1826aa1b0c41e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 4 May 2017 10:56:14 +0800 Subject: [PATCH 416/487] parse comment var 'rows' not init before return --- db_transfer.py | 2 +- shadowsocks/asyncdns.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index 0f3a7c95..2f108705 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -550,8 +550,8 @@ def pull_db_users(self, conn): cur = conn.cursor() try: - cur.execute("SELECT " + ','.join(keys) + " FROM user") rows = [] + cur.execute("SELECT " + ','.join(keys) + " FROM user") for r in cur.fetchall(): d = {} for column in range(len(keys)): diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 3f92e628..01f25226 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -331,6 +331,8 @@ def _parse_hosts(self): with open(etc_path, 'rb') as f: for line in f.readlines(): line = line.strip() + if "#" in line: + line = line[:line.find('#')] parts = line.split() if len(parts) >= 2: ip = parts[0] From 0969937fd1b99ef72b6e959b13834aeb9d486a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 4 May 2017 14:31:24 +0800 Subject: [PATCH 417/487] fix bytes type --- shadowsocks/asyncdns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 01f25226..9713ec15 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -331,8 +331,8 @@ def _parse_hosts(self): with open(etc_path, 'rb') as f: for line in f.readlines(): line = line.strip() - if "#" in line: - line = line[:line.find('#')] + if b"#" in line: + line = line[:line.find(b'#')] parts = line.split() if len(parts) >= 2: ip = parts[0] From 2388802e3f9d8c459022316489acc6b28123f68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 4 May 2017 17:43:51 +0800 Subject: [PATCH 418/487] fix run script --- logrun.sh | 2 +- run.sh | 2 +- shadowsocks/logrun.sh | 2 +- shadowsocks/run.sh | 2 +- shadowsocks/stop.sh | 2 +- stop.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/logrun.sh b/logrun.sh index 0411f51b..94153fe9 100644 --- a/logrun.sh +++ b/logrun.sh @@ -1,6 +1,6 @@ #!/bin/bash cd `dirname $0` -python_ver=$(ls /usr/lib|grep "^python"|tail -1) +python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') ulimit -n 512000 nohup ${python_ver} server.py m>> ssserver.log 2>&1 & diff --git a/run.sh b/run.sh index 16f1f64d..0de8d204 100644 --- a/run.sh +++ b/run.sh @@ -1,6 +1,6 @@ #!/bin/bash cd `dirname $0` -python_ver=$(ls /usr/lib|grep "^python"|tail -1) +python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') ulimit -n 512000 nohup ${python_ver} server.py m>> /dev/null 2>&1 & diff --git a/shadowsocks/logrun.sh b/shadowsocks/logrun.sh index eb3afe27..fc081e1d 100644 --- a/shadowsocks/logrun.sh +++ b/shadowsocks/logrun.sh @@ -1,6 +1,6 @@ #!/bin/bash cd `dirname $0` -python_ver=$(ls /usr/lib|grep "^python"|tail -1) +python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') ulimit -n 512000 nohup ${python_ver} server.py a>> ssserver.log 2>&1 & diff --git a/shadowsocks/run.sh b/shadowsocks/run.sh index 7950c4c9..43720308 100644 --- a/shadowsocks/run.sh +++ b/shadowsocks/run.sh @@ -1,6 +1,6 @@ #!/bin/bash cd `dirname $0` -python_ver=$(ls /usr/lib|grep "^python"|tail -1) +python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') ulimit -n 512000 nohup ${python_ver} server.py a>> /dev/null 2>&1 & diff --git a/shadowsocks/stop.sh b/shadowsocks/stop.sh index 81398909..495b8139 100644 --- a/shadowsocks/stop.sh +++ b/shadowsocks/stop.sh @@ -1,5 +1,5 @@ #!/bin/bash -python_ver=$(ls /usr/lib|grep "^python"|tail -1) +python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') diff --git a/stop.sh b/stop.sh index edca32f3..927e2f07 100644 --- a/stop.sh +++ b/stop.sh @@ -1,5 +1,5 @@ #!/bin/bash -python_ver=$(ls /usr/lib|grep "^python"|tail -1) +python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') From 5cb9445dfd5939cb3fa6ebdbe224ddf779e98e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 5 May 2017 07:29:54 +0800 Subject: [PATCH 419/487] chmod +x --- initcfg.sh | 0 logrun.sh | 0 run.sh | 0 shadowsocks/logrun.sh | 0 shadowsocks/run.sh | 0 shadowsocks/tail.sh | 0 stop.sh | 0 tail.sh | 0 8 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 initcfg.sh mode change 100644 => 100755 logrun.sh mode change 100644 => 100755 run.sh mode change 100644 => 100755 shadowsocks/logrun.sh mode change 100644 => 100755 shadowsocks/run.sh mode change 100644 => 100755 shadowsocks/tail.sh mode change 100644 => 100755 stop.sh mode change 100644 => 100755 tail.sh diff --git a/initcfg.sh b/initcfg.sh old mode 100644 new mode 100755 diff --git a/logrun.sh b/logrun.sh old mode 100644 new mode 100755 diff --git a/run.sh b/run.sh old mode 100644 new mode 100755 diff --git a/shadowsocks/logrun.sh b/shadowsocks/logrun.sh old mode 100644 new mode 100755 diff --git a/shadowsocks/run.sh b/shadowsocks/run.sh old mode 100644 new mode 100755 diff --git a/shadowsocks/tail.sh b/shadowsocks/tail.sh old mode 100644 new mode 100755 diff --git a/stop.sh b/stop.sh old mode 100644 new mode 100755 diff --git a/tail.sh b/tail.sh old mode 100644 new mode 100755 From 26f360c2049556a3c6f6c6cff8f317133c4af47a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 7 May 2017 23:44:59 +0800 Subject: [PATCH 420/487] update mgr --- mujson_mgr.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 320d1277..6c1190e8 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -249,32 +249,24 @@ def main(): '+2': 'tls1.2_ticket_auth_compatible', '2': 'tls1.2_ticket_auth'} fast_set_protocol = {'0': 'origin', - '+ota': 'verify_sha1_compatible', - 'ota': 'verify_sha1', - 'a1': 'auth_sha1', - '+a1': 'auth_sha1_compatible', - 'a2': 'auth_sha1_v2', - '+a2': 'auth_sha1_v2_compatible', 'a4': 'auth_sha1_v4', '+a4': 'auth_sha1_v4_compatible', 'am': 'auth_aes128_md5', 'as': 'auth_aes128_sha1', + 'ca': 'auth_chain_a', } - fast_set_method = {'a0': 'aes-128-cfb', - 'a1': 'aes-192-cfb', - 'a2': 'aes-256-cfb', + fast_set_method = {'0': 'none', + 'a1c': 'aes-128-cfb', + 'a2c': 'aes-192-cfb', + 'a3c': 'aes-256-cfb', 'r': 'rc4-md5', 'r6': 'rc4-md5-6', 'c': 'chacha20', 'ci': 'chacha20-ietf', 's': 'salsa20', - 'b': 'bf-cfb', - 'm0': 'camellia-128-cfb', - 'm1': 'camellia-192-cfb', - 'm2': 'camellia-256-cfb', - 'a0t': 'aes-128-ctr', - 'a1t': 'aes-192-ctr', - 'a2t': 'aes-256-ctr'} + 'a1': 'aes-128-ctr', + 'a2': 'aes-192-ctr', + 'a3': 'aes-256-ctr'} try: optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) for key, value in optlist: From 8aa3fcd207d1716dfc7522e8164df649fddf1a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 9 May 2017 07:39:11 +0800 Subject: [PATCH 421/487] add cymysql setup script --- setup_cymysql.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 setup_cymysql.sh diff --git a/setup_cymysql.sh b/setup_cymysql.sh new file mode 100755 index 00000000..41138850 --- /dev/null +++ b/setup_cymysql.sh @@ -0,0 +1,6 @@ +#!/bin/bash +rm -rf CyMySQL +rm -rf cymysql +git clone https://github.com/nakagami/CyMySQL.git +mv CyMySQL/cymysql ./ +rm -rf CyMySQL From 7599e53a6f87583526d519d79c531e8186cdd8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 9 May 2017 20:02:24 +0800 Subject: [PATCH 422/487] more log for debug --- shadowsocks/tcprelay.py | 59 ++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 3096e00b..c3873739 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -147,7 +147,6 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, # TCP Relay works as either sslocal or ssserver # if is_local, this is sslocal self._is_local = is_local - self._stage = STAGE_INIT self._encrypt_correct = True self._obfs = obfs.obfs(config['obfs']) self._protocol = obfs.obfs(config['protocol']) @@ -211,11 +210,6 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, if is_local: self._chosen_server = self._get_a_server() - fd_to_handlers[local_sock.fileno()] = self - local_sock.setblocking(False) - local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR, self._server) - self.last_activity = 0 self._update_activity() self._server.add_connection(1) @@ -226,6 +220,12 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._udp_send_pack_id = 0 self._udpv6_send_pack_id = 0 + local_sock.setblocking(False) + local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + fd_to_handlers[local_sock.fileno()] = self + loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR, self._server) + self._stage = STAGE_INIT + def __hash__(self): # default __hash__ is id / 16 # we want to eliminate collisions @@ -311,7 +311,6 @@ def _write_to_sock(self, data, sock): # and update the stream to wait for writing if not sock: return False - #logging.debug("_write_to_sock %s %s %s" % (self._remote_sock, sock, self._remote_udp)) uncomplete = False if self._remote_udp and sock == self._remote_sock: try: @@ -832,7 +831,15 @@ def _on_local_read(self): return if obfs_decode[2]: data = self._obfs.server_encode(b'') - self._write_to_sock(data, self._local_sock) + try: + self._write_to_sock(data, self._local_sock) + except Exception as e: + shell.print_exception(e) + if self._config['verbose']: + traceback.print_exc() + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) + self.destroy() + return if obfs_decode[1]: if not self._protocol.obfs.server_info.recv_iv: iv_len = len(self._protocol.obfs.server_info.iv) @@ -859,6 +866,7 @@ def _on_local_read(self): shell.print_exception(e) logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() + return else: return if not data: @@ -870,12 +878,10 @@ def _on_local_read(self): data = self._encryptor.encrypt(data) data = self._obfs.client_encode(data) self._write_to_sock(data, self._remote_sock) - return elif is_local and self._stage == STAGE_INIT: # TODO check auth method self._write_to_sock(b'\x05\00', self._local_sock) self._stage = STAGE_ADDR - return elif self._stage == STAGE_CONNECTING: self._handle_stage_connecting(data) elif (is_local and self._stage == STAGE_ADDR) or \ @@ -1016,36 +1022,34 @@ def handle_event(self, sock, event): if event & eventloop.POLL_ERR: handle = True self._on_remote_error() - if self._stage == STAGE_DESTROYED: - return True - if event & (eventloop.POLL_IN | eventloop.POLL_HUP): + elif event & (eventloop.POLL_IN | eventloop.POLL_HUP): if not self.speed_tester_d.isExceed(): if not self._server.speed_tester_d(self._user_id).isExceed(): handle = True self._on_remote_read(sock == self._remote_sock) - if self._stage == STAGE_DESTROYED: - return True - if event & eventloop.POLL_OUT: + elif event & eventloop.POLL_OUT: handle = True self._on_remote_write() elif sock == self._local_sock: if event & eventloop.POLL_ERR: handle = True self._on_local_error() - if self._stage == STAGE_DESTROYED: - return True - if event & (eventloop.POLL_IN | eventloop.POLL_HUP): + elif event & (eventloop.POLL_IN | eventloop.POLL_HUP): if not self.speed_tester_u.isExceed(): if not self._server.speed_tester_u(self._user_id).isExceed(): handle = True self._on_local_read() - if self._stage == STAGE_DESTROYED: - return True - if event & eventloop.POLL_OUT: + elif event & eventloop.POLL_OUT: handle = True self._on_local_write() else: logging.warn('unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) + try: + self._loop.remove(sock) + except Exception as e: + shell.print_exception(e) + del self._fd_to_handlers[sock.fileno()] + sock.close() return handle @@ -1079,16 +1083,16 @@ def destroy(self): try: self._loop.remove(self._remote_sock) except Exception as e: - pass + shell.print_exception(e) del self._fd_to_handlers[self._remote_sock.fileno()] self._remote_sock.close() self._remote_sock = None if self._remote_sock_v6: - logging.debug('destroying remote') + logging.debug('destroying remote_v6') try: self._loop.remove(self._remote_sock_v6) except Exception as e: - pass + shell.print_exception(e) del self._fd_to_handlers[self._remote_sock_v6.fileno()] self._remote_sock_v6.close() self._remote_sock_v6 = None @@ -1334,6 +1338,7 @@ def handle_event(self, sock, fd, event): if event & eventloop.POLL_ERR: # TODO raise Exception('server_socket error') + handler = None try: logging.debug('accept') conn = self._server_socket.accept() @@ -1351,11 +1356,15 @@ def handle_event(self, sock, fd, event): shell.print_exception(e) if self._config['verbose']: traceback.print_exc() + if handler: + handler.destroy() else: if sock: handler = self._fd_to_handlers.get(fd, None) if handler: handler.handle_event(sock, event) + else: + logging.warn('unknown fd') else: logging.warn('poll removed fd') From 45faa93dd36e2b65a226c9ded2075e67d9341f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 9 May 2017 21:46:37 +0800 Subject: [PATCH 423/487] close unknown socket --- shadowsocks/tcprelay.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c3873739..4ffbc0da 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1048,7 +1048,10 @@ def handle_event(self, sock, event): self._loop.remove(sock) except Exception as e: shell.print_exception(e) - del self._fd_to_handlers[sock.fileno()] + try: + del self._fd_to_handlers[sock.fileno()] + except Exception as e: + shell.print_exception(e) sock.close() return handle @@ -1084,7 +1087,10 @@ def destroy(self): self._loop.remove(self._remote_sock) except Exception as e: shell.print_exception(e) - del self._fd_to_handlers[self._remote_sock.fileno()] + try: + del self._fd_to_handlers[self._remote_sock.fileno()] + except Exception as e: + shell.print_exception(e) self._remote_sock.close() self._remote_sock = None if self._remote_sock_v6: @@ -1093,13 +1099,22 @@ def destroy(self): self._loop.remove(self._remote_sock_v6) except Exception as e: shell.print_exception(e) - del self._fd_to_handlers[self._remote_sock_v6.fileno()] + try: + del self._fd_to_handlers[self._remote_sock_v6.fileno()] + except Exception as e: + shell.print_exception(e) self._remote_sock_v6.close() self._remote_sock_v6 = None if self._local_sock: logging.debug('destroying local') - self._loop.remove(self._local_sock) - del self._fd_to_handlers[self._local_sock.fileno()] + try: + self._loop.remove(self._local_sock) + except Exception as e: + shell.print_exception(e) + try: + del self._fd_to_handlers[self._local_sock.fileno()] + except Exception as e: + shell.print_exception(e) self._local_sock.close() self._local_sock = None if self._obfs: @@ -1365,6 +1380,11 @@ def handle_event(self, sock, fd, event): handler.handle_event(sock, event) else: logging.warn('unknown fd') + try: + self._eventloop.remove(sock) + except Exception as e: + shell.print_exception(e) + sock.close() else: logging.warn('poll removed fd') From be696555a7b3bff1465be7f624eaf8f755c8bb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 9 May 2017 23:20:37 +0800 Subject: [PATCH 424/487] new speed tester --- shadowsocks/eventloop.py | 2 +- shadowsocks/tcprelay.py | 95 ++++++++++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 6e2f22f2..d2bbeb65 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -214,7 +214,7 @@ def run(self): if handler is not None: handler = handler[1] try: - handle = handle or handler.handle_event(sock, fd, event) + handle = handler.handle_event(sock, fd, event) or handle except (OSError, IOError) as e: shell.print_exception(e) now = time.time() diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 4ffbc0da..235505fc 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -28,7 +28,6 @@ import random import platform import threading -from collections import deque from shadowsocks import encrypt, obfs, eventloop, shell, common, lru_cache from shadowsocks.common import pre_parse_header, parse_header @@ -91,7 +90,7 @@ WAIT_STATUS_WRITING = 2 WAIT_STATUS_READWRITING = WAIT_STATUS_READING | WAIT_STATUS_WRITING -NETWORK_MTU = 1492 +NETWORK_MTU = 1500 TCP_MSS = NETWORK_MTU - 40 BUF_SIZE = 32 * 1024 UDP_MAX_BUF_SIZE = 65536 @@ -99,8 +98,7 @@ class SpeedTester(object): def __init__(self, max_speed = 0): self.max_speed = max_speed * 1024 - self.timeout = 1 - self._cache = deque() + self.last_time = time.time() self.sum_len = 0 def update_limit(self, max_speed): @@ -108,19 +106,21 @@ def update_limit(self, max_speed): def add(self, data_len): if self.max_speed > 0: - self._cache.append((time.time(), data_len)) + cut_t = time.time() + self.sum_len -= (cut_t - self.last_time) * self.max_speed + if self.sum_len < 0: + self.sum_len = 0 + self.last_time = cut_t self.sum_len += data_len def isExceed(self): if self.max_speed > 0: - if self.sum_len > 0: - cut_t = time.time() - t = max(cut_t - self._cache[0][0], 0.01) - speed = self.sum_len / t - if self._cache[0][0] + self.timeout < cut_t: - self.sum_len -= self._cache[0][1] - self._cache.popleft() - return speed >= self.max_speed + cut_t = time.time() + self.sum_len -= (cut_t - self.last_time) * self.max_speed + if self.sum_len < 0: + self.sum_len = 0 + self.last_time = cut_t + return self.sum_len >= self.max_speed return False class TCPRelayHandler(object): @@ -132,6 +132,9 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._local_sock = local_sock self._remote_sock = None self._remote_sock_v6 = None + self._local_sock_fd = None + self._remote_sock_fd = None + self._remotev6_sock_fd = None self._remote_udp = False self._config = config self._dns_resolver = dns_resolver @@ -216,13 +219,16 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._server.stat_add(self._client_address[0], 1) self.speed_tester_u = SpeedTester(config.get("speed_limit_per_con", 0)) self.speed_tester_d = SpeedTester(config.get("speed_limit_per_con", 0)) + self._recv_u_max_size = BUF_SIZE + self._recv_d_max_size = BUF_SIZE self._recv_pack_id = 0 self._udp_send_pack_id = 0 self._udpv6_send_pack_id = 0 local_sock.setblocking(False) local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - fd_to_handlers[local_sock.fileno()] = self + self._local_sock_fd = local_sock.fileno() + fd_to_handlers[self._local_sock_fd] = self loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR, self._server) self._stage = STAGE_INIT @@ -695,13 +701,15 @@ def _create_remote_socket(self, ip, port): raise Exception('Port %d is in forbidden list, reject' % sa[1]) remote_sock = socket.socket(af, socktype, proto) self._remote_sock = remote_sock - self._fd_to_handlers[remote_sock.fileno()] = self + self._remote_sock_fd = remote_sock.fileno() + self._fd_to_handlers[self._remote_sock_fd] = self if self._remote_udp: af, socktype, proto, canonname, sa = addrs_v6[0] remote_sock_v6 = socket.socket(af, socktype, proto) self._remote_sock_v6 = remote_sock_v6 - self._fd_to_handlers[remote_sock_v6.fileno()] = self + self._remotev6_sock_fd = remote_sock_v6.fileno() + self._fd_to_handlers[self._remotev6_sock_fd] = self remote_sock.setblocking(False) if self._remote_udp: @@ -784,10 +792,16 @@ def _handle_dns_resolved(self, result, error): logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() - def _get_read_size(self, sock, recv_buffer_size): + def _get_read_size(self, sock, recv_buffer_size, up): if self._overhead == 0: return recv_buffer_size buffer_size = len(sock.recv(recv_buffer_size, socket.MSG_PEEK)) + if up: + buffer_size = min(buffer_size, self._recv_u_max_size) + self._recv_u_max_size = min(self._recv_u_max_size + TCP_MSS, BUF_SIZE) + else: + buffer_size = min(buffer_size, self._recv_d_max_size) + self._recv_d_max_size = min(self._recv_d_max_size + TCP_MSS, BUF_SIZE) if buffer_size == recv_buffer_size: return buffer_size s = buffer_size % self._tcp_mss + self._overhead @@ -802,7 +816,7 @@ def _on_local_read(self): return is_local = self._is_local if is_local: - recv_buffer_size = self._get_read_size(self._local_sock, self._recv_buffer_size) + recv_buffer_size = self._get_read_size(self._local_sock, self._recv_buffer_size, True) else: recv_buffer_size = BUF_SIZE data = None @@ -914,7 +928,7 @@ def _on_remote_read(self, is_remote_sock): if self._is_local: recv_buffer_size = BUF_SIZE else: - recv_buffer_size = self._get_read_size(self._remote_sock, self._recv_buffer_size) + recv_buffer_size = self._get_read_size(self._remote_sock, self._recv_buffer_size, False) data = self._remote_sock.recv(recv_buffer_size) self._recv_pack_id += 1 except (OSError, IOError) as e: @@ -1008,7 +1022,7 @@ def _on_remote_error(self): logging.error("remote error, exception from %s:%d" % (self._client_address[0], self._client_address[1])) self.destroy() - def handle_event(self, sock, event): + def handle_event(self, sock, fd, event): # handle all events in this handler and dispatch them to methods handle = False if self._stage == STAGE_DESTROYED: @@ -1023,10 +1037,11 @@ def handle_event(self, sock, event): handle = True self._on_remote_error() elif event & (eventloop.POLL_IN | eventloop.POLL_HUP): - if not self.speed_tester_d.isExceed(): - if not self._server.speed_tester_d(self._user_id).isExceed(): - handle = True - self._on_remote_read(sock == self._remote_sock) + if not self.speed_tester_d.isExceed() and not self._server.speed_tester_d(self._user_id).isExceed(): + handle = True + self._on_remote_read(sock == self._remote_sock) + else: + self._recv_d_max_size = TCP_MSS elif event & eventloop.POLL_OUT: handle = True self._on_remote_write() @@ -1035,10 +1050,11 @@ def handle_event(self, sock, event): handle = True self._on_local_error() elif event & (eventloop.POLL_IN | eventloop.POLL_HUP): - if not self.speed_tester_u.isExceed(): - if not self._server.speed_tester_u(self._user_id).isExceed(): - handle = True - self._on_local_read() + if not self.speed_tester_u.isExceed() and not self._server.speed_tester_u(self._user_id).isExceed(): + handle = True + self._on_local_read() + else: + self._recv_u_max_size = TCP_MSS elif event & eventloop.POLL_OUT: handle = True self._on_local_write() @@ -1049,7 +1065,7 @@ def handle_event(self, sock, event): except Exception as e: shell.print_exception(e) try: - del self._fd_to_handlers[sock.fileno()] + del self._fd_to_handlers[fd] except Exception as e: shell.print_exception(e) sock.close() @@ -1088,7 +1104,8 @@ def destroy(self): except Exception as e: shell.print_exception(e) try: - del self._fd_to_handlers[self._remote_sock.fileno()] + if self._remote_sock_fd is not None: + del self._fd_to_handlers[self._remote_sock_fd] except Exception as e: shell.print_exception(e) self._remote_sock.close() @@ -1100,7 +1117,8 @@ def destroy(self): except Exception as e: shell.print_exception(e) try: - del self._fd_to_handlers[self._remote_sock_v6.fileno()] + if self._remotev6_sock_fd is not None: + del self._fd_to_handlers[self._remotev6_sock_fd] except Exception as e: shell.print_exception(e) self._remote_sock_v6.close() @@ -1112,7 +1130,8 @@ def destroy(self): except Exception as e: shell.print_exception(e) try: - del self._fd_to_handlers[self._local_sock.fileno()] + if self._local_sock_fd is not None: + del self._fd_to_handlers[self._local_sock_fd] except Exception as e: shell.print_exception(e) self._local_sock.close() @@ -1346,6 +1365,7 @@ def _close_tcp_client(self, client): def handle_event(self, sock, fd, event): # handle events and dispatch to handlers + handle = False if sock: logging.log(shell.VERBOSE_LEVEL, 'fd %d %s', fd, eventloop.EVENT_NAMES.get(event, event)) @@ -1354,6 +1374,7 @@ def handle_event(self, sock, fd, event): # TODO raise Exception('server_socket error') handler = None + handle = True try: logging.debug('accept') conn = self._server_socket.accept() @@ -1377,9 +1398,10 @@ def handle_event(self, sock, fd, event): if sock: handler = self._fd_to_handlers.get(fd, None) if handler: - handler.handle_event(sock, event) + handle = handler.handle_event(sock, fd, event) else: logging.warn('unknown fd') + handle = True try: self._eventloop.remove(sock) except Exception as e: @@ -1387,6 +1409,13 @@ def handle_event(self, sock, fd, event): sock.close() else: logging.warn('poll removed fd') + handle = True + if fd in self._fd_to_handlers: + try: + del self._fd_to_handlers[fd] + except Exception as e: + shell.print_exception(e) + return handle def handle_periodic(self): if self._closed: From a1f56d1233ab213ae86310eefe38944cfc4f3266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 10 May 2017 10:27:29 +0800 Subject: [PATCH 425/487] refine log --- shadowsocks/asyncdns.py | 2 +- shadowsocks/tcprelay.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 9713ec15..797704e3 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -364,7 +364,7 @@ def _call_callback(self, hostname, ip, error=None): callback((hostname, ip), error) else: callback((hostname, None), - Exception('unknown hostname %s' % hostname)) + Exception('unable to parse hostname %s' % hostname)) if hostname in self._hostname_to_cb: del self._hostname_to_cb[hostname] if hostname in self._hostname_status: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 235505fc..414bf43f 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -587,7 +587,7 @@ def _handle_stage_addr(self, ogn_data, data): # just trim VER CMD RSV data = data[3:] else: - logging.error('unknown command %d', cmd) + logging.error('invalid command %d', cmd) self.destroy() return From 5e14119b41b7139ff7a5d570003ea3ff4136ff41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 10 May 2017 20:10:23 +0800 Subject: [PATCH 426/487] removefd --- shadowsocks/eventloop.py | 4 ++++ shadowsocks/tcprelay.py | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index d2bbeb65..341620ef 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -176,6 +176,10 @@ def remove(self, f): del self._fdmap[fd] self._impl.unregister(fd) + def removefd(self, fd): + del self._fdmap[fd] + self._impl.unregister(fd) + def add_periodic(self, callback): self._periodic_callbacks.append(callback) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 414bf43f..6ddc5317 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1061,7 +1061,7 @@ def handle_event(self, sock, fd, event): else: logging.warn('unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) try: - self._loop.remove(sock) + self._loop.removefd(fd) except Exception as e: shell.print_exception(e) try: @@ -1100,7 +1100,7 @@ def destroy(self): if self._remote_sock: logging.debug('destroying remote') try: - self._loop.remove(self._remote_sock) + self._loop.removefd(self._remote_sock_fd) except Exception as e: shell.print_exception(e) try: @@ -1113,7 +1113,7 @@ def destroy(self): if self._remote_sock_v6: logging.debug('destroying remote_v6') try: - self._loop.remove(self._remote_sock_v6) + self._loop.removefd(self._remotev6_sock_fd) except Exception as e: shell.print_exception(e) try: @@ -1126,7 +1126,7 @@ def destroy(self): if self._local_sock: logging.debug('destroying local') try: - self._loop.remove(self._local_sock) + self._loop.removefd(self._local_sock_fd) except Exception as e: shell.print_exception(e) try: @@ -1403,7 +1403,7 @@ def handle_event(self, sock, fd, event): logging.warn('unknown fd') handle = True try: - self._eventloop.remove(sock) + self._eventloop.removefd(fd) except Exception as e: shell.print_exception(e) sock.close() From 19e2e7422dc5428766ae9f6ddbfe2ea8c462d569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 11 May 2017 00:07:34 +0800 Subject: [PATCH 427/487] fd instead --- shadowsocks/tcprelay.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 6ddc5317..693dca21 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1031,8 +1031,7 @@ def handle_event(self, sock, fd, event): if self._user is not None and self._user not in self._server.server_users: self.destroy() return True - # order is important - if sock == self._remote_sock or sock == self._remote_sock_v6: + if fd == self._remote_sock_fd or fd == self._remotev6_sock_fd: if event & eventloop.POLL_ERR: handle = True self._on_remote_error() @@ -1045,7 +1044,7 @@ def handle_event(self, sock, fd, event): elif event & eventloop.POLL_OUT: handle = True self._on_remote_write() - elif sock == self._local_sock: + elif fd == self._local_sock_fd: if event & eventloop.POLL_ERR: handle = True self._on_local_error() @@ -1204,6 +1203,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self._config['fast_open'] = False server_socket.listen(config.get('max_connect', 1024)) self._server_socket = server_socket + self._server_socket_fd = server_socket.fileno() self._stat_counter = stat_counter self._stat_callback = stat_callback @@ -1420,7 +1420,7 @@ def handle_event(self, sock, fd, event): def handle_periodic(self): if self._closed: if self._server_socket: - self._eventloop.remove(self._server_socket) + self._eventloop.removefd(self._server_socket_fd) self._server_socket.close() self._server_socket = None logging.info('closed TCP port %d', self._listen_port) @@ -1434,7 +1434,7 @@ def close(self, next_tick=False): if not next_tick: if self._eventloop: self._eventloop.remove_periodic(self.handle_periodic) - self._eventloop.remove(self._server_socket) + self._eventloop.removefd(self._server_socket_fd) self._server_socket.close() for handler in list(self._fd_to_handlers.values()): handler.destroy() From 709e67afe47fad1ce2de46a49d40b1105acd9e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 11 May 2017 20:51:24 +0800 Subject: [PATCH 428/487] bump --- CHANGES | 13 +++++++++++++ mujson_mgr.py | 4 ++-- shadowsocks/version.py | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 072bb1d7..1f59c415 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,16 @@ +3.3.0 2017-05-11 +- connect_log include local addr & port +- fix auth_chain_a UDP bug +- add "additional_ports_only" +- add interface legendsockssr +- run with newest python version +- parse comment in hosts +- update mujson_mgr +- add cymysql setup script +- new speed tester +- fix leaks +- bugs fixed + 3.2.0 2017-04-27 - add auth_chain_a - remove auth_aes128, auth_sha1, auth_sha1_v2, verify_simple, auth_simple, verify_sha1 diff --git a/mujson_mgr.py b/mujson_mgr.py index 6c1190e8..39e6b6e2 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -249,8 +249,8 @@ def main(): '+2': 'tls1.2_ticket_auth_compatible', '2': 'tls1.2_ticket_auth'} fast_set_protocol = {'0': 'origin', - 'a4': 'auth_sha1_v4', - '+a4': 'auth_sha1_v4_compatible', + 's4': 'auth_sha1_v4', + '+s4': 'auth_sha1_v4_compatible', 'am': 'auth_aes128_md5', 'as': 'auth_aes128_sha1', 'ca': 'auth_chain_a', diff --git a/shadowsocks/version.py b/shadowsocks/version.py index eea9c31d..786000fe 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.2.0 2017-04-27' + return '3.3.0 2017-05-11' From 6c162b13c417d5bf1f18f989a247c613062b54a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 12 May 2017 17:13:58 +0800 Subject: [PATCH 429/487] fix stop script --- shadowsocks/stop.sh | 2 +- stop.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/stop.sh b/shadowsocks/stop.sh index 495b8139..d7d29589 100644 --- a/shadowsocks/stop.sh +++ b/shadowsocks/stop.sh @@ -1,5 +1,5 @@ #!/bin/bash python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) -eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py a" | awk '{print "kill "$2}') diff --git a/stop.sh b/stop.sh index 927e2f07..56567daa 100755 --- a/stop.sh +++ b/stop.sh @@ -1,5 +1,5 @@ #!/bin/bash python_ver=$(ls /usr/bin|grep -e "^python[23]\.[1-9]\+$"|tail -1) -eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') +eval $(ps -ef | grep "[0-9] ${python_ver} server\\.py m" | awk '{print "kill "$2}') From abe4ac3b90ece169a8e5970cdad485116cd976f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 12 May 2017 18:11:38 +0800 Subject: [PATCH 430/487] Async DNS query under UDP --- shadowsocks/common.py | 2 +- shadowsocks/tcprelay.py | 80 +++++++++++++------ shadowsocks/udprelay.py | 173 +++++++++++++++++++++++----------------- 3 files changed, 156 insertions(+), 99 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index fdb22437..40e46cf7 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -236,7 +236,7 @@ def parse_header(data): 'encryption method' % addrtype) if dest_addr is None: return None - return connecttype, to_bytes(dest_addr), dest_port, header_length + return connecttype, addrtype, to_bytes(dest_addr), dest_port, header_length class IPNetwork(object): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 693dca21..f8160520 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -123,6 +123,29 @@ def isExceed(self): return self.sum_len >= self.max_speed return False +class UDPAsyncDNSHandler(object): + def __init__(self, params): + self.params = params + self.remote_addr = None + self.call_back = None + + def resolve(self, dns_resolver, remote_addr, call_back): + self.call_back = call_back + self.remote_addr = remote_addr + dns_resolver.resolve(remote_addr[0], self._handle_dns_resolved) + + def _handle_dns_resolved(self, result, error): + if error: + logging.error("%s when resolve DNS" % (error,)) #drop + return + if result: + ip = result[1] + if ip: + if self.call_back: + self.call_back(self.params, self.remote_addr, ip) + return + logging.warning("can't resolve %s" % (self.remote_addr,)) + class TCPRelayHandler(object): def __init__(self, server, fd_to_handlers, loop, local_sock, config, dns_resolver, is_local): @@ -344,26 +367,12 @@ def _write_to_sock(self, data, sock): header_result = parse_header(data) if header_result is None: continue - connecttype, dest_addr, dest_port, header_length = header_result - addrs = socket.getaddrinfo(dest_addr, dest_port, 0, - socket.SOCK_DGRAM, socket.SOL_UDP) - if addrs: - af, socktype, proto, canonname, server_addr = addrs[0] - data = data[header_length:] - if af == socket.AF_INET6: - self._remote_sock_v6.sendto(data, (server_addr[0], dest_port)) - if self._udpv6_send_pack_id == 0: - addr, port = self._remote_sock_v6.getsockname()[:2] - common.connect_log('UDPv6 sendto %s:%d from %s:%d by user %d' % - (server_addr[0], dest_port, addr, port, self._user_id)) - self._udpv6_send_pack_id += 1 - else: - sock.sendto(data, (server_addr[0], dest_port)) - if self._udp_send_pack_id == 0: - addr, port = sock.getsockname()[:2] - common.connect_log('UDP sendto %s:%d from %s:%d by user %d' % - (server_addr[0], dest_port, addr, port, self._user_id)) - self._udp_send_pack_id += 1 + connecttype, addrtype, dest_addr, dest_port, header_length = header_result + if (addrtype & 7) == 3: + handler = UDPAsyncDNSHandler(data[header_length:]) + handler.resolve(self._dns_resolver, (dest_addr, dest_port), self._handle_server_dns_resolved) + else: + return self._handle_server_dns_resolved(data[header_length:], (dest_addr, dest_port), dest_addr) except Exception as e: #trace = traceback.format_exc() @@ -426,6 +435,31 @@ def _write_to_sock(self, data, sock): logging.error('write_all_to_sock:unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) return True + def _handle_server_dns_resolved(self, data, remote_addr, server_addr): + try: + addrs = socket.getaddrinfo(server_addr, remote_addr[1], 0, socket.SOCK_DGRAM, socket.SOL_UDP) + if not addrs: # drop + return + af, socktype, proto, canonname, sa = addrs[0] + if af == socket.AF_INET6: + self._remote_sock_v6.sendto(data, (server_addr, remote_addr[1])) + if self._udpv6_send_pack_id == 0: + addr, port = self._remote_sock_v6.getsockname()[:2] + common.connect_log('UDPv6 sendto %s(%s):%d from %s:%d by user %d' % + (common.to_str(remote_addr[0]), common.to_str(server_addr), remote_addr[1], addr, port, self._user_id)) + self._udpv6_send_pack_id += 1 + else: + self._remote_sock.sendto(data, (server_addr, remote_addr[1])) + if self._udp_send_pack_id == 0: + addr, port = self._remote_sock.getsockname()[:2] + common.connect_log('UDP sendto %s(%s):%d from %s:%d by user %d' % + (common.to_str(remote_addr[0]), common.to_str(server_addr), remote_addr[1], addr, port, self._user_id)) + self._udp_send_pack_id += 1 + return True + except Exception as e: + shell.print_exception(e) + logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) + def _get_redirect_host(self, client_address, ogn_data): host_list = self._redir_list or ["*#0.0.0.0:0"] @@ -601,7 +635,7 @@ def _handle_stage_addr(self, ogn_data, data): header_result = parse_header(data) if header_result is not None: try: - common.to_str(header_result[1]) + common.to_str(header_result[2]) except Exception as e: header_result = None if header_result is None: @@ -613,7 +647,7 @@ def _handle_stage_addr(self, ogn_data, data): server_info.buffer_size = self._recv_buffer_size server_info = self._protocol.get_server_info() server_info.buffer_size = self._recv_buffer_size - connecttype, remote_addr, remote_port, header_length = header_result + connecttype, addrtype, remote_addr, remote_port, header_length = header_result if connecttype != 0: pass #common.connect_log('UDP over TCP by user %d' % @@ -771,7 +805,7 @@ def _handle_dns_resolved(self, result, error): raise e addr, port = self._remote_sock.getsockname()[:2] common.connect_log('TCP connecting %s(%s):%d from %s:%d by user %d' % - (self._remote_address[0], remote_addr, remote_port, addr, port, self._user_id)) + (common.to_str(self._remote_address[0]), common.to_str(remote_addr), remote_port, addr, port, self._user_id)) self._loop.add(remote_sock, eventloop.POLL_ERR | eventloop.POLL_OUT, diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index c1539791..8271d1ea 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -123,11 +123,33 @@ RSP_STATE_DISCONNECT = b"\x04" RSP_STATE_REDIRECT = b"\x05" +class UDPAsyncDNSHandler(object): + def __init__(self, params): + self.params = params + self.remote_addr = None + self.call_back = None + + def resolve(self, dns_resolver, remote_addr, call_back): + self.call_back = call_back + self.remote_addr = remote_addr + dns_resolver.resolve(remote_addr[0], self._handle_dns_resolved) + + def _handle_dns_resolved(self, result, error): + if error: + logging.error("%s when resolve DNS" % (error,)) #drop + return + if result: + ip = result[1] + if ip: + if self.call_back: + self.call_back(*self.params, self.remote_addr, None, ip, True) + return + logging.warning("can't resolve %s" % (self.remote_addr,)) + def client_key(source_addr, server_af): # notice this is server af, not dest af return '%s:%s:%d' % (source_addr[0], source_addr[1], server_af) - class UDPRelay(object): def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_counter=None): self._config = config @@ -154,7 +176,7 @@ def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_coun self._cache_dns_client = lru_cache.LRUCache(timeout=10, close_callback=self._close_client_pair) self._client_fd_to_server_addr = {} - self._dns_cache = lru_cache.LRUCache(timeout=1800) + #self._dns_cache = lru_cache.LRUCache(timeout=1800) self._eventloop = None self._closed = False self.server_transfer_ul = 0 @@ -375,97 +397,98 @@ def _handle_server(self): if header_result is None: self._handel_protocol_error(r_addr, ogn_data) return - connecttype, dest_addr, dest_port, header_length = header_result + connecttype, addrtype, dest_addr, dest_port, header_length = header_result if self._is_local: - connecttype = 3 + addrtype = 3 server_addr, server_port = self._get_a_server() else: server_addr, server_port = dest_addr, dest_port - if (connecttype & 7) == 3: - addrs = self._dns_cache.get(server_addr, None) + if (addrtype & 7) == 3: + handler = UDPAsyncDNSHandler((data, r_addr, uid, header_length)) + handler.resolve(self._dns_resolver, (server_addr, server_port), self._handle_server_dns_resolved) + else: + self._handle_server_dns_resolved(data, r_addr, uid, header_length, (server_addr, server_port), None, server_addr, False) + + def _handle_server_dns_resolved(self, data, r_addr, uid, header_length, remote_addr, addrs, server_addr, dns_resolved): + try: + server_port = remote_addr[1] if addrs is None: - # TODO async getaddrinfo addrs = socket.getaddrinfo(server_addr, server_port, 0, - socket.SOCK_DGRAM, socket.SOL_UDP) - if not addrs: - # drop - return - else: - self._dns_cache[server_addr] = addrs - else: - addrs = socket.getaddrinfo(server_addr, server_port, 0, - socket.SOCK_DGRAM, socket.SOL_UDP) - if not addrs: - # drop + socket.SOCK_DGRAM, socket.SOL_UDP) + if not addrs: # drop return + af, socktype, proto, canonname, sa = addrs[0] + server_addr = sa[0] + key = client_key(r_addr, af) + client_pair = self._cache.get(key, None) + if client_pair is None: + client_pair = self._cache_dns_client.get(key, None) + if client_pair is None: + if self._forbidden_iplist: + if common.to_str(sa[0]) in self._forbidden_iplist: + logging.debug('IP %s is in forbidden list, drop' % common.to_str(sa[0])) + # drop + return + if self._forbidden_portset: + if sa[1] in self._forbidden_portset: + logging.debug('Port %d is in forbidden list, reject' % sa[1]) + # drop + return + client = socket.socket(af, socktype, proto) + client_uid = uid + client.setblocking(False) + self._socket_bind_addr(client, af) + is_dns = False + if len(data) > header_length + 13 and data[header_length + 4 : header_length + 12] == b"\x00\x01\x00\x00\x00\x00\x00\x00": + is_dns = True + else: + pass + if sa[1] == 53 and is_dns: #DNS + logging.debug("DNS query %s from %s:%d" % (common.to_str(sa[0]), r_addr[0], r_addr[1])) + self._cache_dns_client[key] = (client, uid) + else: + self._cache[key] = (client, uid) + self._client_fd_to_server_addr[client.fileno()] = (r_addr, af) - af, socktype, proto, canonname, sa = addrs[0] - key = client_key(r_addr, af) - client_pair = self._cache.get(key, None) - if client_pair is None: - client_pair = self._cache_dns_client.get(key, None) - if client_pair is None: - if self._forbidden_iplist: - if common.to_str(sa[0]) in self._forbidden_iplist: - logging.debug('IP %s is in forbidden list, drop' % common.to_str(sa[0])) - # drop - return - if self._forbidden_portset: - if sa[1] in self._forbidden_portset: - logging.debug('Port %d is in forbidden list, reject' % sa[1]) - # drop - return - client = socket.socket(af, socktype, proto) - client_uid = uid - client.setblocking(False) - self._socket_bind_addr(client, af) - is_dns = False - if len(data) > 20 and data[11:19] == b"\x00\x01\x00\x00\x00\x00\x00\x00": - is_dns = True - else: - pass - if sa[1] == 53 and is_dns: #DNS - logging.debug("DNS query %s from %s:%d" % (common.to_str(sa[0]), r_addr[0], r_addr[1])) - self._cache_dns_client[key] = (client, uid) - else: - self._cache[key] = (client, uid) - self._client_fd_to_server_addr[client.fileno()] = (r_addr, af) - - self._sockets.add(client.fileno()) - self._eventloop.add(client, eventloop.POLL_IN, self) + self._sockets.add(client.fileno()) + self._eventloop.add(client, eventloop.POLL_IN, self) - logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) + logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) - if uid is None: - user_id = self._listen_port + if uid is None: + user_id = self._listen_port + else: + user_id = struct.unpack(' Date: Fri, 12 May 2017 18:52:22 +0800 Subject: [PATCH 431/487] let py2 works --- shadowsocks/udprelay.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 8271d1ea..ece08a43 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -142,7 +142,7 @@ def _handle_dns_resolved(self, result, error): ip = result[1] if ip: if self.call_back: - self.call_back(*self.params, self.remote_addr, None, ip, True) + self.call_back(self.remote_addr, None, ip, True, *self.params) return logging.warning("can't resolve %s" % (self.remote_addr,)) @@ -409,9 +409,9 @@ def _handle_server(self): handler = UDPAsyncDNSHandler((data, r_addr, uid, header_length)) handler.resolve(self._dns_resolver, (server_addr, server_port), self._handle_server_dns_resolved) else: - self._handle_server_dns_resolved(data, r_addr, uid, header_length, (server_addr, server_port), None, server_addr, False) + self._handle_server_dns_resolved((server_addr, server_port), None, server_addr, False, data, r_addr, uid, header_length) - def _handle_server_dns_resolved(self, data, r_addr, uid, header_length, remote_addr, addrs, server_addr, dns_resolved): + def _handle_server_dns_resolved(self, remote_addr, addrs, server_addr, dns_resolved, data, r_addr, uid, header_length): try: server_port = remote_addr[1] if addrs is None: From 67065a212f43993c4d4db8da034b8571f1d98dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 12 May 2017 19:00:07 +0800 Subject: [PATCH 432/487] chmod +x --- shadowsocks/stop.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 shadowsocks/stop.sh diff --git a/shadowsocks/stop.sh b/shadowsocks/stop.sh old mode 100644 new mode 100755 From 96200d7d76afa3d116c2cfa30a23bea4d32421d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 14 May 2017 16:42:48 +0800 Subject: [PATCH 433/487] fix key size --- shadowsocks/crypto/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/table.py b/shadowsocks/crypto/table.py index fd6aa94c..60c2f245 100644 --- a/shadowsocks/crypto/table.py +++ b/shadowsocks/crypto/table.py @@ -74,7 +74,7 @@ def update(self, data): ciphers = { 'none': (16, 0, NoneCipher), - 'table': (0, 0, TableCipher) + 'table': (16, 0, TableCipher) } From 85b6930d7d0f54c59be3dbb39f5568392377304b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 17 May 2017 19:59:48 +0800 Subject: [PATCH 434/487] fix old version of OpenSSL ignore warning --- shadowsocks/crypto/openssl.py | 3 +-- shadowsocks/shell.py | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/shadowsocks/crypto/openssl.py b/shadowsocks/crypto/openssl.py index 24264f18..aa117593 100644 --- a/shadowsocks/crypto/openssl.py +++ b/shadowsocks/crypto/openssl.py @@ -89,8 +89,7 @@ def __init__(self, cipher_name, key, iv, op): self._ctx = None if not loaded: load_openssl() - cipher_name = common.to_bytes(cipher_name) - cipher = libcrypto.EVP_get_cipherbyname(cipher_name) + cipher = libcrypto.EVP_get_cipherbyname(common.to_bytes(cipher_name)) if not cipher: cipher = load_cipher(cipher_name) if not cipher: diff --git a/shadowsocks/shell.py b/shadowsocks/shell.py index f0e0cb01..6246d98a 100755 --- a/shadowsocks/shell.py +++ b/shadowsocks/shell.py @@ -111,12 +111,6 @@ def check_config(config, is_local): if config.get('server', '') in ['127.0.0.1', 'localhost']: logging.warning('warning: server set to listen on %s:%s, are you sure?' % (to_str(config['server']), config['server_port'])) - if (config.get('method', '') or '').lower() == 'table': - logging.warning('warning: table is not safe; please use a safer cipher, ' - 'like AES-256-CFB') - if (config.get('method', '') or '').lower() == 'rc4': - logging.warning('warning: RC4 is not safe; please use a safer cipher, ' - 'like AES-256-CFB') if config.get('timeout', 300) < 100: logging.warning('warning: your timeout %d seems too short' % int(config.get('timeout'))) From d6e1f73c9f0e6353123ed0572b792f44656cbe08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 17 May 2017 20:13:17 +0800 Subject: [PATCH 435/487] fix old version of OpenSSL --- shadowsocks/crypto/openssl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/shadowsocks/crypto/openssl.py b/shadowsocks/crypto/openssl.py index aa117593..0a8ca53f 100644 --- a/shadowsocks/crypto/openssl.py +++ b/shadowsocks/crypto/openssl.py @@ -67,8 +67,6 @@ def load_openssl(): def load_cipher(cipher_name): func_name = 'EVP_' + cipher_name.replace('-', '_') - if bytes != str: - func_name = str(func_name, 'utf-8') cipher = getattr(libcrypto, func_name, None) if cipher: cipher.restype = c_void_p From 1dc6c4be4503f43fde64747750eca26be0730a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 18 May 2017 17:40:50 +0800 Subject: [PATCH 436/487] http reply --- shadowsocks/tcprelay.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f8160520..faa4af68 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -29,7 +29,7 @@ import platform import threading -from shadowsocks import encrypt, obfs, eventloop, shell, common, lru_cache +from shadowsocks import encrypt, obfs, eventloop, shell, common, lru_cache, version from shadowsocks.common import pre_parse_header, parse_header # we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time @@ -655,6 +655,14 @@ def _handle_stage_addr(self, ogn_data, data): else: common.connect_log('TCP request %s:%d by user %d' % (common.to_str(remote_addr), remote_port, self._user_id)) + if remote_addr == b'0.0.0.0' and remote_port == 80: + backdata = b'HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n' + common.to_bytes(version.version()) + backdata = self._protocol.server_pre_encrypt(backdata) + backdata = self._encryptor.encrypt(backdata) + backdata = self._obfs.server_encode(backdata) + self._write_to_sock(backdata, self._local_sock) + self.destroy() + return self._remote_address = (common.to_str(remote_addr), remote_port) self._remote_udp = (connecttype != 0) # pause reading From 6e3fdd0d6212017bb6ce82132edfeabf8dd74884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 18 May 2017 17:43:37 +0800 Subject: [PATCH 437/487] bump --- CHANGES | 6 ++++++ shadowsocks/tcprelay.py | 2 +- shadowsocks/version.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 1f59c415..2c3bd3fe 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +3.3.1 2017-05-18 +- fix stop script +- Async DNS query under UDP +- fix old version of OpenSSL +- http reply + 3.3.0 2017-05-11 - connect_log include local addr & port - fix auth_chain_a UDP bug diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index faa4af68..7dfb4456 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -656,7 +656,7 @@ def _handle_stage_addr(self, ogn_data, data): common.connect_log('TCP request %s:%d by user %d' % (common.to_str(remote_addr), remote_port, self._user_id)) if remote_addr == b'0.0.0.0' and remote_port == 80: - backdata = b'HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n' + common.to_bytes(version.version()) + backdata = b'HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nShadowsocksR ' + common.to_bytes(version.version()) backdata = self._protocol.server_pre_encrypt(backdata) backdata = self._encryptor.encrypt(backdata) backdata = self._obfs.server_encode(backdata) diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 786000fe..748404a5 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.3.0 2017-05-11' + return '3.3.1 2017-05-18' From 7a68e4e637821e21ebcf28f5299e78a5ce67d00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 19 May 2017 00:50:47 +0800 Subject: [PATCH 438/487] more reply addr --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7dfb4456..21c5b3b0 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -655,7 +655,7 @@ def _handle_stage_addr(self, ogn_data, data): else: common.connect_log('TCP request %s:%d by user %d' % (common.to_str(remote_addr), remote_port, self._user_id)) - if remote_addr == b'0.0.0.0' and remote_port == 80: + if (remote_addr == b'0.0.0.0' or remote_addr == b'255.0.0.255' or remote_addr == b'fe80::ffff') and remote_port == 80: backdata = b'HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nShadowsocksR ' + common.to_bytes(version.version()) backdata = self._protocol.server_pre_encrypt(backdata) backdata = self._encryptor.encrypt(backdata) From 86de572caf04b58acbba08161a5ded97b2ac6d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 19 May 2017 01:38:40 +0800 Subject: [PATCH 439/487] revert http reply --- shadowsocks/tcprelay.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 21c5b3b0..65df4c84 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -655,14 +655,6 @@ def _handle_stage_addr(self, ogn_data, data): else: common.connect_log('TCP request %s:%d by user %d' % (common.to_str(remote_addr), remote_port, self._user_id)) - if (remote_addr == b'0.0.0.0' or remote_addr == b'255.0.0.255' or remote_addr == b'fe80::ffff') and remote_port == 80: - backdata = b'HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nShadowsocksR ' + common.to_bytes(version.version()) - backdata = self._protocol.server_pre_encrypt(backdata) - backdata = self._encryptor.encrypt(backdata) - backdata = self._obfs.server_encode(backdata) - self._write_to_sock(backdata, self._local_sock) - self.destroy() - return self._remote_address = (common.to_str(remote_addr), remote_port) self._remote_udp = (connecttype != 0) # pause reading From 24623b896dfeadc80a09f0f4a0a3493df0237c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 20 May 2017 14:50:27 +0800 Subject: [PATCH 440/487] fix str type --- shadowsocks/server.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index f4dfdd5f..c18ad1cc 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -29,7 +29,7 @@ sys.path.insert(0, os.path.join(file_path, '../')) from shadowsocks import shell, daemon, eventloop, tcprelay, udprelay, \ - asyncdns, manager + asyncdns, manager, common def main(): @@ -86,16 +86,16 @@ def main(): bindv6 = config.get("out_bindv6", '') if type(password_obfs) == list: password = password_obfs[0] - obfs = password_obfs[1] + obfs = common.to_str(password_obfs[1]) if len(password_obfs) > 2: - protocol = password_obfs[2] + protocol = common.to_str(password_obfs[2]) elif type(password_obfs) == dict: password = password_obfs.get('password', config_password) - method = password_obfs.get('method', method) - protocol = password_obfs.get('protocol', protocol) - protocol_param = password_obfs.get('protocol_param', protocol_param) - obfs = password_obfs.get('obfs', obfs) - obfs_param = password_obfs.get('obfs_param', obfs_param) + method = common.to_str(password_obfs.get('method', method)) + protocol = common.to_str(password_obfs.get('protocol', protocol)) + protocol_param = common.to_str(password_obfs.get('protocol_param', protocol_param)) + obfs = common.to_str(password_obfs.get('obfs', obfs)) + obfs_param = common.to_str(password_obfs.get('obfs_param', obfs_param)) bind = password_obfs.get('out_bind', bind) bindv6 = password_obfs.get('out_bindv6', bindv6) else: From 3b0200581ca9487e5796511dd38029dc4f926c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 20 May 2017 18:39:26 +0800 Subject: [PATCH 441/487] refine error detector --- shadowsocks/obfsplugin/obfs_tls.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index e4899114..c2354785 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -241,12 +241,13 @@ def server_decode(self, buf): if not match_begin(buf, b'\x16\x03\x01'): return self.decode_error_return(ogn_buf) buf = buf[3:] - if struct.unpack('>H', buf[:2])[0] > len(buf) - 2: + header_len = struct.unpack('>H', buf[:2])[0] + if header_len > len(buf) - 2: return (b'', False, False) - self.recv_buffer = self.recv_buffer[struct.unpack('>H', buf[:2])[0] + 5:] + self.recv_buffer = self.recv_buffer[header_len + 5:] self.handshake_status = 2 - buf = buf[2:] + buf = buf[2:header_len + 2] if not match_begin(buf, b'\x01\x00'): #client hello logging.info("tls_auth not client hello message") return self.decode_error_return(ogn_buf) From 542a2e62d929278999277692371c4b0198a2c6b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 20 May 2017 19:04:51 +0800 Subject: [PATCH 442/487] bump 3.3.2 --- CHANGES | 4 ++++ shadowsocks/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 2c3bd3fe..44581249 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +3.3.2 2017-05-20 +- revert http reply +- refine tls1.2_ticket_auth error detector + 3.3.1 2017-05-18 - fix stop script - Async DNS query under UDP diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 748404a5..937e9445 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.3.1 2017-05-18' + return '3.3.2 2017-05-20' From 77628acde4ee0d1bc3300cb389cdb93c7b50c55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 20 May 2017 19:44:35 +0800 Subject: [PATCH 443/487] allow fast sending request --- shadowsocks/obfsplugin/obfs_tls.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index c2354785..ecb87d67 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -164,7 +164,7 @@ def client_decode(self, buf): def server_encode(self, buf): if self.handshake_status == -1: return buf - if self.handshake_status == 8: + if (self.handshake_status & 8) == 8: ret = b'' while len(buf) > 2048: size = min(struct.unpack('>H', os.urandom(2))[0] % 4096 + 100, len(buf)) @@ -173,18 +173,20 @@ def server_encode(self, buf): if len(buf) > 0: ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf return ret - self.handshake_status = 3 + self.handshake_status |= 8 data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"c02f000005ff01000100") data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello - data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data + data = b"\x16" + self.tls_version + struct.pack('>H', len(data)) + data if random.randint(0, 8) < 1: ticket = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 164) * 2 + 64) - ticket = struct.pack('>H', len(ticket) + 4) + b"\x04\x00" + struct.pack('>H', len(ticket)) + ticket = struct.pack('>H', len(ticket) + 4) + b"\x04\x00" + struct.pack('>H', len(ticket)) + ticket data += b"\x16" + self.tls_version + ticket #New session ticket data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec finish_len = random.choice([32, 40]) data += b"\x16" + self.tls_version + struct.pack('>H', finish_len) + os.urandom(finish_len - 10) #Finished data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10] + if buf: + data += self.server_encode(buf) return data def decode_error_return(self, buf): @@ -198,7 +200,7 @@ def server_decode(self, buf): if self.handshake_status == -1: return (buf, True, False) - if self.handshake_status == 8: + if (self.handshake_status & 4) == 4: ret = b'' self.recv_buffer += buf while len(self.recv_buffer) > 5: @@ -212,11 +214,11 @@ def server_decode(self, buf): self.recv_buffer = self.recv_buffer[size+5:] return (ret, True, False) - if self.handshake_status == 3: + if (self.handshake_status & 1) == 1: self.recv_buffer += buf buf = self.recv_buffer verify = buf - if len(buf) < 43: + if len(buf) < 11: raise Exception('server_decode data error') if not match_begin(buf, b"\x14" + self.tls_version + b"\x00\x01\x01"): #ChangeCipherSpec raise Exception('server_decode data error') @@ -225,12 +227,14 @@ def server_decode(self, buf): raise Exception('server_decode data error') verify_len = struct.unpack('>H', buf[3:5])[0] + 1 # 11 - 10 if len(verify) < verify_len + 10: - raise Exception('server_decode data error') + return (b'', False, False) if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]: raise Exception('server_decode data error') self.recv_buffer = verify[verify_len + 10:] - self.handshake_status = 8 - return self.server_decode(b'') + status = self.handshake_status + self.handshake_status |= 4 + ret = self.server_decode(b'') + return ret; #raise Exception("handshake data = %s" % (binascii.hexlify(buf))) self.recv_buffer += buf @@ -246,7 +250,7 @@ def server_decode(self, buf): return (b'', False, False) self.recv_buffer = self.recv_buffer[header_len + 5:] - self.handshake_status = 2 + self.handshake_status = 1 buf = buf[2:header_len + 2] if not match_begin(buf, b'\x01\x00'): #client hello logging.info("tls_auth not client hello message") @@ -289,6 +293,9 @@ def server_decode(self, buf): return self.decode_error_return(ogn_buf) self.server_info.data.client_data.sweep() self.server_info.data.client_data[verifyid[:22]] = sessionid + if len(self.recv_buffer) >= 11: + ret = self.server_decode(b'') + return (ret[0], True, True) # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back) return (b'', False, True) From c1364c3afef69da369d3b9dcc1d5638f9ec18eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 27 May 2017 13:06:35 +0800 Subject: [PATCH 444/487] init user_id --- shadowsocks/udprelay.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index ece08a43..c69b6ed9 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -412,6 +412,7 @@ def _handle_server(self): self._handle_server_dns_resolved((server_addr, server_port), None, server_addr, False, data, r_addr, uid, header_length) def _handle_server_dns_resolved(self, remote_addr, addrs, server_addr, dns_resolved, data, r_addr, uid, header_length): + user_id = self._listen_port try: server_port = remote_addr[1] if addrs is None: @@ -457,9 +458,7 @@ def _handle_server_dns_resolved(self, remote_addr, addrs, server_addr, dns_resol logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) - if uid is None: - user_id = self._listen_port - else: + if uid is not None: user_id = struct.unpack(' Date: Sat, 27 May 2017 21:28:43 +0800 Subject: [PATCH 445/487] fix overhead size --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 65df4c84..3a615126 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -176,7 +176,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._encrypt_correct = True self._obfs = obfs.obfs(config['obfs']) self._protocol = obfs.obfs(config['protocol']) - self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._encryptor.iv_len() self._recv_buffer_size = BUF_SIZE - self._overhead server_info = obfs.server_info(server.obfs_data) @@ -641,7 +641,7 @@ def _handle_stage_addr(self, ogn_data, data): if header_result is None: data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) - self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._encryptor.iv_len() self._recv_buffer_size = BUF_SIZE - self._overhead server_info = self._obfs.get_server_info() server_info.buffer_size = self._recv_buffer_size From a3315bcd6a350494f7d4b3dcf27458ab881ea8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 28 May 2017 15:00:02 +0800 Subject: [PATCH 446/487] Revert "fix overhead size" This reverts commit 587453c881458a8a650274d44fa1cb8243ab83d7. --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 3a615126..65df4c84 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -176,7 +176,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._encrypt_correct = True self._obfs = obfs.obfs(config['obfs']) self._protocol = obfs.obfs(config['protocol']) - self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._encryptor.iv_len() + self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) self._recv_buffer_size = BUF_SIZE - self._overhead server_info = obfs.server_info(server.obfs_data) @@ -641,7 +641,7 @@ def _handle_stage_addr(self, ogn_data, data): if header_result is None: data = self._handel_protocol_error(self._client_address, ogn_data) header_result = parse_header(data) - self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + self._encryptor.iv_len() + self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) self._recv_buffer_size = BUF_SIZE - self._overhead server_info = self._obfs.get_server_info() server_info.buffer_size = self._recv_buffer_size From 92722c2ea96c0fa59a4c8e182d3993cdf018f361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 29 May 2017 08:42:57 +0800 Subject: [PATCH 447/487] refine _get_read_size --- shadowsocks/tcprelay.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 65df4c84..661f3599 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -832,15 +832,15 @@ def _get_read_size(self, sock, recv_buffer_size, up): buffer_size = len(sock.recv(recv_buffer_size, socket.MSG_PEEK)) if up: buffer_size = min(buffer_size, self._recv_u_max_size) - self._recv_u_max_size = min(self._recv_u_max_size + TCP_MSS, BUF_SIZE) + self._recv_u_max_size = min(self._recv_u_max_size + self._tcp_mss - self._overhead, BUF_SIZE) else: buffer_size = min(buffer_size, self._recv_d_max_size) - self._recv_d_max_size = min(self._recv_d_max_size + TCP_MSS, BUF_SIZE) + self._recv_d_max_size = min(self._recv_d_max_size + self._tcp_mss - self._overhead, BUF_SIZE) if buffer_size == recv_buffer_size: return buffer_size - s = buffer_size % self._tcp_mss + self._overhead - if s > self._tcp_mss: - return buffer_size - (s - self._tcp_mss) + frame_size = self._tcp_mss - self._overhead + if buffer_size > frame_size: + buffer_size = int(buffer_size / frame_size) * frame_size return buffer_size def _on_local_read(self): @@ -1074,7 +1074,7 @@ def handle_event(self, sock, fd, event): handle = True self._on_remote_read(sock == self._remote_sock) else: - self._recv_d_max_size = TCP_MSS + self._recv_d_max_size = self._tcp_mss - self._overhead elif event & eventloop.POLL_OUT: handle = True self._on_remote_write() @@ -1087,7 +1087,7 @@ def handle_event(self, sock, fd, event): handle = True self._on_local_read() else: - self._recv_u_max_size = TCP_MSS + self._recv_u_max_size = self._tcp_mss - self._overhead elif event & eventloop.POLL_OUT: handle = True self._on_local_write() From f31a81665cb0c12878fd6baf3d1c85bb88494c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 30 May 2017 21:32:51 +0800 Subject: [PATCH 448/487] refine UDP over TCP --- shadowsocks/tcprelay.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 661f3599..f4012767 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -347,15 +347,11 @@ def _write_to_sock(self, data, sock): #logging.info('UDP over TCP sendto %d %s' % (len(data), binascii.hexlify(data))) while len(self._udp_data_send_buffer) > 6: length = struct.unpack('>H', self._udp_data_send_buffer[:2])[0] - if length >= 0xff00: - length = struct.unpack('>H', self._udp_data_send_buffer[1:3])[0] + 0xff00 if length > len(self._udp_data_send_buffer): break data = self._udp_data_send_buffer[:length] - if length >= 0xff00: - data = data[1:] self._udp_data_send_buffer = self._udp_data_send_buffer[length:] frag = common.ord(data[2]) @@ -953,10 +949,7 @@ def _on_remote_read(self, is_remote_sock): ip = socket.inet_pton(socket.AF_INET6, addr[0]) data = b'\x00\x04' + ip + port + data size = len(data) + 2 - if size >= 0xff00: - data = common.chr(0xff) + struct.pack('>H', size - 0xff00 + 1) + data - else: - data = struct.pack('>H', size) + data + data = struct.pack('>H', size) + data #logging.info('UDP over TCP recvfrom %s:%d %d bytes to %s:%d' % (addr[0], addr[1], len(data), self._client_address[0], self._client_address[1])) else: if self._is_local: From b89931f9fcf66df85f9b61031506b9d034d96b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 31 May 2017 17:46:07 +0800 Subject: [PATCH 449/487] fix cmp in Python 3 --- shadowsocks/common.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 40e46cf7..f40b5faa 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -297,6 +297,12 @@ def __contains__(self, addr): def __cmp__(self, other): return cmp(self.addrs_str, other.addrs_str) + def __eq__(self, other): + return self.addrs_str == other.addrs_str + + def __ne__(self, other): + return self.addrs_str != other.addrs_str + class PortRange(object): def __init__(self, range_str): self.range_str = to_str(range_str) @@ -328,6 +334,12 @@ def __contains__(self, val): def __cmp__(self, other): return cmp(self.range_str, other.range_str) + def __eq__(self, other): + return self.range_str == other.range_str + + def __ne__(self, other): + return self.range_str != other.range_str + def test_inet_conv(): ipv4 = b'8.8.4.4' b = inet_pton(socket.AF_INET, ipv4) From 2cf5f1982e3c007fcf3e21eb775372a8f55c86e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 1 Jun 2017 00:11:15 +0800 Subject: [PATCH 450/487] add DNS cache --- shadowsocks/common.py | 31 +++++++++++++++++++++++++++++++ shadowsocks/tcprelay.py | 31 ++++++------------------------- shadowsocks/udprelay.py | 33 ++++++--------------------------- 3 files changed, 43 insertions(+), 52 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index f40b5faa..f5666d3d 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -24,6 +24,8 @@ import binascii import re +from shadowsocks import lru_cache + def compat_ord(s): if type(s) == int: return s @@ -340,6 +342,35 @@ def __eq__(self, other): def __ne__(self, other): return self.range_str != other.range_str +class UDPAsyncDNSHandler(object): + dns_cache = lru_cache.LRUCache(timeout=1800) + def __init__(self, params): + self.params = params + self.remote_addr = None + self.call_back = None + + def resolve(self, dns_resolver, remote_addr, call_back): + if remote_addr in UDPAsyncDNSHandler.dns_cache: + if call_back: + call_back(remote_addr, None, UDPAsyncDNSHandler.dns_cache[remote_addr], True, *self.params) + else: + self.call_back = call_back + self.remote_addr = remote_addr + dns_resolver.resolve(remote_addr[0], self._handle_dns_resolved) + UDPAsyncDNSHandler.dns_cache.sweep() + + def _handle_dns_resolved(self, result, error): + if error: + logging.error("%s when resolve DNS" % (error,)) #drop + return + if result: + ip = result[1] + if ip: + if self.call_back: + self.call_back(self.remote_addr, None, ip, True, *self.params) + return + logging.warning("can't resolve %s" % (self.remote_addr,)) + def test_inet_conv(): ipv4 = b'8.8.4.4' b = inet_pton(socket.AF_INET, ipv4) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f4012767..328b48d1 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -123,29 +123,6 @@ def isExceed(self): return self.sum_len >= self.max_speed return False -class UDPAsyncDNSHandler(object): - def __init__(self, params): - self.params = params - self.remote_addr = None - self.call_back = None - - def resolve(self, dns_resolver, remote_addr, call_back): - self.call_back = call_back - self.remote_addr = remote_addr - dns_resolver.resolve(remote_addr[0], self._handle_dns_resolved) - - def _handle_dns_resolved(self, result, error): - if error: - logging.error("%s when resolve DNS" % (error,)) #drop - return - if result: - ip = result[1] - if ip: - if self.call_back: - self.call_back(self.params, self.remote_addr, ip) - return - logging.warning("can't resolve %s" % (self.remote_addr,)) - class TCPRelayHandler(object): def __init__(self, server, fd_to_handlers, loop, local_sock, config, dns_resolver, is_local): @@ -365,8 +342,12 @@ def _write_to_sock(self, data, sock): continue connecttype, addrtype, dest_addr, dest_port, header_length = header_result if (addrtype & 7) == 3: - handler = UDPAsyncDNSHandler(data[header_length:]) - handler.resolve(self._dns_resolver, (dest_addr, dest_port), self._handle_server_dns_resolved) + af = common.is_ip(dest_addr) + if af == False: + handler = common.UDPAsyncDNSHandler(data[header_length:]) + handler.resolve(self._dns_resolver, (dest_addr, dest_port), self._handle_server_dns_resolved) + else: + return self._handle_server_dns_resolved(data[header_length:], (dest_addr, dest_port), dest_addr) else: return self._handle_server_dns_resolved(data[header_length:], (dest_addr, dest_port), dest_addr) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index c69b6ed9..23eb6d98 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -123,29 +123,6 @@ RSP_STATE_DISCONNECT = b"\x04" RSP_STATE_REDIRECT = b"\x05" -class UDPAsyncDNSHandler(object): - def __init__(self, params): - self.params = params - self.remote_addr = None - self.call_back = None - - def resolve(self, dns_resolver, remote_addr, call_back): - self.call_back = call_back - self.remote_addr = remote_addr - dns_resolver.resolve(remote_addr[0], self._handle_dns_resolved) - - def _handle_dns_resolved(self, result, error): - if error: - logging.error("%s when resolve DNS" % (error,)) #drop - return - if result: - ip = result[1] - if ip: - if self.call_back: - self.call_back(self.remote_addr, None, ip, True, *self.params) - return - logging.warning("can't resolve %s" % (self.remote_addr,)) - def client_key(source_addr, server_af): # notice this is server af, not dest af return '%s:%s:%d' % (source_addr[0], source_addr[1], server_af) @@ -406,8 +383,12 @@ def _handle_server(self): server_addr, server_port = dest_addr, dest_port if (addrtype & 7) == 3: - handler = UDPAsyncDNSHandler((data, r_addr, uid, header_length)) - handler.resolve(self._dns_resolver, (server_addr, server_port), self._handle_server_dns_resolved) + af = common.is_ip(server_addr) + if af == False: + handler = common.UDPAsyncDNSHandler((data, r_addr, uid, header_length)) + handler.resolve(self._dns_resolver, (server_addr, server_port), self._handle_server_dns_resolved) + else: + self._handle_server_dns_resolved((server_addr, server_port), None, server_addr, False, data, r_addr, uid, header_length) else: self._handle_server_dns_resolved((server_addr, server_port), None, server_addr, False, data, r_addr, uid, header_length) @@ -645,7 +626,6 @@ def handle_periodic(self): if self._closed: self._cache.clear(0) self._cache_dns_client.clear(0) - #self._dns_cache.sweep() if self._eventloop: self._eventloop.remove_periodic(self.handle_periodic) self._eventloop.remove(self._server_socket) @@ -657,7 +637,6 @@ def handle_periodic(self): before_sweep_size = len(self._sockets) self._cache.sweep() self._cache_dns_client.sweep() - #self._dns_cache.sweep() if before_sweep_size != len(self._sockets): logging.debug('UDP port %5d sockets %d' % (self._listen_port, len(self._sockets))) self._sweep_timeout() From 280c762760937dd3ecffc8c8c79359d80dde9bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 3 Jun 2017 13:51:35 +0800 Subject: [PATCH 451/487] add tls1.2_ticket_fastauth --- shadowsocks/obfsplugin/obfs_tls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index ecb87d67..19471fc0 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -41,6 +41,8 @@ def create_tls_ticket_auth_obfs(method): obfs_map = { 'tls1.2_ticket_auth': (create_tls_ticket_auth_obfs,), 'tls1.2_ticket_auth_compatible': (create_tls_ticket_auth_obfs,), + 'tls1.2_ticket_fastauth': (create_tls_ticket_auth_obfs,), + 'tls1.2_ticket_fastauth_compatible': (create_tls_ticket_auth_obfs,), } def match_begin(str1, str2): From 7c3d4e01b12b6ddf894dd7c7b5097df5d90b4d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 3 Jun 2017 13:55:07 +0800 Subject: [PATCH 452/487] bump 3.3.3 --- CHANGES | 5 +++++ shadowsocks/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 44581249..d5fe72fb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +3.3.3 2017-06-03 +- add DNS cache +- add tls1.2_ticket_fastauth +- fix bugs + 3.3.2 2017-05-20 - revert http reply - refine tls1.2_ticket_auth error detector diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 937e9445..74eac22b 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.3.2 2017-05-20' + return '3.3.3 2017-06-03' From 8826f515b63541d66827e560ee6d823a4ce488b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 3 Jun 2017 21:47:17 +0800 Subject: [PATCH 453/487] fix async DNS --- shadowsocks/common.py | 4 ++-- shadowsocks/tcprelay.py | 6 +++--- shadowsocks/udprelay.py | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index f5666d3d..ffaacfbe 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -352,7 +352,7 @@ def __init__(self, params): def resolve(self, dns_resolver, remote_addr, call_back): if remote_addr in UDPAsyncDNSHandler.dns_cache: if call_back: - call_back(remote_addr, None, UDPAsyncDNSHandler.dns_cache[remote_addr], True, *self.params) + call_back(remote_addr, UDPAsyncDNSHandler.dns_cache[remote_addr], self.params) else: self.call_back = call_back self.remote_addr = remote_addr @@ -367,7 +367,7 @@ def _handle_dns_resolved(self, result, error): ip = result[1] if ip: if self.call_back: - self.call_back(self.remote_addr, None, ip, True, *self.params) + self.call_back(self.remote_addr, ip, self.params) return logging.warning("can't resolve %s" % (self.remote_addr,)) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 328b48d1..f9f337f0 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -347,9 +347,9 @@ def _write_to_sock(self, data, sock): handler = common.UDPAsyncDNSHandler(data[header_length:]) handler.resolve(self._dns_resolver, (dest_addr, dest_port), self._handle_server_dns_resolved) else: - return self._handle_server_dns_resolved(data[header_length:], (dest_addr, dest_port), dest_addr) + return self._handle_server_dns_resolved((dest_addr, dest_port), dest_addr, data[header_length:]) else: - return self._handle_server_dns_resolved(data[header_length:], (dest_addr, dest_port), dest_addr) + return self._handle_server_dns_resolved((dest_addr, dest_port), dest_addr, data[header_length:]) except Exception as e: #trace = traceback.format_exc() @@ -412,7 +412,7 @@ def _write_to_sock(self, data, sock): logging.error('write_all_to_sock:unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) return True - def _handle_server_dns_resolved(self, data, remote_addr, server_addr): + def _handle_server_dns_resolved(self, remote_addr, server_addr, data): try: addrs = socket.getaddrinfo(server_addr, remote_addr[1], 0, socket.SOCK_DGRAM, socket.SOL_UDP) if not addrs: # drop diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 23eb6d98..4eaf2541 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -388,17 +388,17 @@ def _handle_server(self): handler = common.UDPAsyncDNSHandler((data, r_addr, uid, header_length)) handler.resolve(self._dns_resolver, (server_addr, server_port), self._handle_server_dns_resolved) else: - self._handle_server_dns_resolved((server_addr, server_port), None, server_addr, False, data, r_addr, uid, header_length) + self._handle_server_dns_resolved((server_addr, server_port), server_addr, (data, r_addr, uid, header_length)) else: - self._handle_server_dns_resolved((server_addr, server_port), None, server_addr, False, data, r_addr, uid, header_length) + self._handle_server_dns_resolved((server_addr, server_port), server_addr, (data, r_addr, uid, header_length)) - def _handle_server_dns_resolved(self, remote_addr, addrs, server_addr, dns_resolved, data, r_addr, uid, header_length): + def _handle_server_dns_resolved(self, remote_addr, server_addr, params): + data, r_addr, uid, header_length = params user_id = self._listen_port try: server_port = remote_addr[1] - if addrs is None: - addrs = socket.getaddrinfo(server_addr, server_port, 0, - socket.SOCK_DGRAM, socket.SOL_UDP) + addrs = socket.getaddrinfo(server_addr, server_port, 0, + socket.SOCK_DGRAM, socket.SOL_UDP) if not addrs: # drop return af, socktype, proto, canonname, sa = addrs[0] From 47c6e16833536632fb06420982226ffb6951a295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 3 Jun 2017 23:06:12 +0800 Subject: [PATCH 454/487] fix a mem leak --- shadowsocks/common.py | 9 ++++----- shadowsocks/tcprelay.py | 9 ++++++--- shadowsocks/udprelay.py | 8 +++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index ffaacfbe..c4484c04 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -352,7 +352,7 @@ def __init__(self, params): def resolve(self, dns_resolver, remote_addr, call_back): if remote_addr in UDPAsyncDNSHandler.dns_cache: if call_back: - call_back(remote_addr, UDPAsyncDNSHandler.dns_cache[remote_addr], self.params) + call_back("", remote_addr, UDPAsyncDNSHandler.dns_cache[remote_addr], self.params) else: self.call_back = call_back self.remote_addr = remote_addr @@ -362,14 +362,13 @@ def resolve(self, dns_resolver, remote_addr, call_back): def _handle_dns_resolved(self, result, error): if error: logging.error("%s when resolve DNS" % (error,)) #drop - return + return self.call_back(error, self.remote_addr, None, self.params) if result: ip = result[1] if ip: - if self.call_back: - self.call_back(self.remote_addr, ip, self.params) - return + return self.call_back("", self.remote_addr, ip, self.params) logging.warning("can't resolve %s" % (self.remote_addr,)) + return self.call_back("fail to resolve", self.remote_addr, None, self.params) def test_inet_conv(): ipv4 = b'8.8.4.4' diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f9f337f0..cd6d0da7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -347,9 +347,9 @@ def _write_to_sock(self, data, sock): handler = common.UDPAsyncDNSHandler(data[header_length:]) handler.resolve(self._dns_resolver, (dest_addr, dest_port), self._handle_server_dns_resolved) else: - return self._handle_server_dns_resolved((dest_addr, dest_port), dest_addr, data[header_length:]) + return self._handle_server_dns_resolved("", (dest_addr, dest_port), dest_addr, data[header_length:]) else: - return self._handle_server_dns_resolved((dest_addr, dest_port), dest_addr, data[header_length:]) + return self._handle_server_dns_resolved("", (dest_addr, dest_port), dest_addr, data[header_length:]) except Exception as e: #trace = traceback.format_exc() @@ -412,7 +412,10 @@ def _write_to_sock(self, data, sock): logging.error('write_all_to_sock:unknown socket from %s:%d' % (self._client_address[0], self._client_address[1])) return True - def _handle_server_dns_resolved(self, remote_addr, server_addr, data): + def _handle_server_dns_resolved(self, error, remote_addr, server_addr, data): + if error: + self.destroy() + return try: addrs = socket.getaddrinfo(server_addr, remote_addr[1], 0, socket.SOCK_DGRAM, socket.SOL_UDP) if not addrs: # drop diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 4eaf2541..a7bbf3cf 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -388,11 +388,13 @@ def _handle_server(self): handler = common.UDPAsyncDNSHandler((data, r_addr, uid, header_length)) handler.resolve(self._dns_resolver, (server_addr, server_port), self._handle_server_dns_resolved) else: - self._handle_server_dns_resolved((server_addr, server_port), server_addr, (data, r_addr, uid, header_length)) + self._handle_server_dns_resolved("", (server_addr, server_port), server_addr, (data, r_addr, uid, header_length)) else: - self._handle_server_dns_resolved((server_addr, server_port), server_addr, (data, r_addr, uid, header_length)) + self._handle_server_dns_resolved("", (server_addr, server_port), server_addr, (data, r_addr, uid, header_length)) - def _handle_server_dns_resolved(self, remote_addr, server_addr, params): + def _handle_server_dns_resolved(self, error, remote_addr, server_addr, params): + if error: + return data, r_addr, uid, header_length = params user_id = self._listen_port try: From f22ef99d8573f80e7d3baad7d62c3224823b74e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 3 Jun 2017 23:09:09 +0800 Subject: [PATCH 455/487] do not destroy --- shadowsocks/tcprelay.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index cd6d0da7..3322b521 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -414,7 +414,6 @@ def _write_to_sock(self, data, sock): def _handle_server_dns_resolved(self, error, remote_addr, server_addr, data): if error: - self.destroy() return try: addrs = socket.getaddrinfo(server_addr, remote_addr[1], 0, socket.SOCK_DGRAM, socket.SOL_UDP) From 82f8fef28aa300ea3ad3e09a4742da5108b98e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 4 Jun 2017 18:34:12 +0800 Subject: [PATCH 456/487] allow set speed limit in runtime --- db_transfer.py | 2 +- mujson_mgr.py | 6 +++++- server_pool.py | 3 +-- shadowsocks/tcprelay.py | 31 +++++++++++++++++++++++++------ shadowsocks/udprelay.py | 15 ++++++++------- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 2f108705..e7583817 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -123,7 +123,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): continue if allow: - allow_users[port] = passwd + allow_users[port] = cfg if 'protocol' in cfg and 'protocol_param' in cfg and common.to_str(cfg['protocol']) in obfs.mu_protocol(): if '#' in common.to_str(cfg['protocol_param']): mu_servers[port] = passwd diff --git a/mujson_mgr.py b/mujson_mgr.py index 39e6b6e2..47e91a6b 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -239,7 +239,7 @@ def print_server_help(): def main(): - shortopts = 'adeclu:i:p:k:O:o:G:g:m:t:f:h' + shortopts = 'adeclu:i:p:k:O:o:G:g:m:t:f:hs:S:' longopts = ['help'] action = None user = {} @@ -302,6 +302,10 @@ def main(): user['obfs_param'] = value elif key == '-G': user['protocol_param'] = value + elif key == '-s': + user['speed_limit_per_con'] = int(value) + elif key == '-S': + user['speed_limit_per_user'] = int(value) elif key == '-m': if value in fast_set_method: user['method'] = fast_set_method[value] diff --git a/server_pool.py b/server_pool.py index 2e6a85bc..d159817a 100644 --- a/server_pool.py +++ b/server_pool.py @@ -28,6 +28,7 @@ from shadowsocks import shell, eventloop, tcprelay, udprelay, asyncdns, common import threading import sys +import traceback from socket import * from configloader import load_config, get_config @@ -80,12 +81,10 @@ def _loop(loop, dns_resolver, mgr): loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) - import traceback traceback.print_exc() os.exit(0) except Exception as e: logging.error(e) - import traceback traceback.print_exc() def server_is_run(self, port): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 3322b521..7aae7e46 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -273,6 +273,11 @@ def _create_encryptor(self, config): def _update_user(self, user): self._user = user self._user_id = struct.unpack(' Date: Mon, 5 Jun 2017 11:42:11 +0800 Subject: [PATCH 457/487] fix KeyError --- db_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_transfer.py b/db_transfer.py index e7583817..3a1441d1 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -139,7 +139,7 @@ def del_server_out_of_bound_safe(self, last_rows, rows): if not cfgchange and port in ServerPool.get_instance().tcp_ipv6_servers_pool: relay = ServerPool.get_instance().tcp_ipv6_servers_pool[port] for name in merge_config_keys: - if name in cfg and not self.cmp(cfg[name], relay._config[name]): + if (name in cfg) and ((name not in relay._config) or not self.cmp(cfg[name], relay._config[name])): cfgchange = True break From 71625e6415faa55642090bfecac71f1ce3513471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 16 Jun 2017 21:12:02 +0800 Subject: [PATCH 458/487] refine Dockerfile --- Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7e7383f6..a35c1fb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,10 @@ FROM alpine ENV SERVER_ADDR 0.0.0.0 ENV SERVER_PORT 51348 -ENV PASSWORD= +ENV PASSWORD psw ENV METHOD aes-128-ctr ENV PROTOCOL auth_aes128_md5 +ENV PROTOCOLPARAM 32 ENV OBFS tls1.2_ticket_auth_compatible ENV TIMEOUT 300 ENV DNS_ADDR 8.8.8.8 @@ -20,13 +21,13 @@ RUN apk update \ -RUN wget --no-check-certificate https://github.com/breakwa11/shadowsocks/archive/manyuser.zip -O /tmp/manyuser.zip \ +RUN wget --no-check-certificate https://github.com/shadowsocksr/shadowsocksr/archive/manyuser.zip -O /tmp/manyuser.zip \ && unzip -d /tmp /tmp/manyuser.zip \ - && mv /tmp/shadowsocksr-manyuser/shadowsocks ~/shadowsocks \ + && mv /tmp/shadowsocksr-manyuser ~/shadowsocksr \ && rm -rf /tmp/* -WORKDIR ~/shadowsocks +WORKDIR ~/shadowsocksr/shadowsocks -CMD python ~/shadowsocks/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS +CMD python ~/shadowsocksr/shadowsocks/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS -G $PROTOCOLPARAM From b655222857b4b58263be03d398265e5b03550237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 16 Jun 2017 21:32:52 +0800 Subject: [PATCH 459/487] fix http_post under py3 --- shadowsocks/obfsplugin/http_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index f84beea1..67f97b6f 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -219,7 +219,7 @@ def __init__(self, method): super(http_post, self).__init__(method) def boundary(self): - return b''.join([random.choice(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") for i in range(32)]) + return b''.join([chr(random.choice(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")) for i in range(32)]) def client_encode(self, buf): if self.has_sent_header: From 7b62f9f5302271bf5f030c7aed1d0df3813c7cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 16 Jun 2017 21:45:56 +0800 Subject: [PATCH 460/487] fix http_post under py3 --- shadowsocks/obfsplugin/http_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 67f97b6f..30d7ca91 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -219,7 +219,7 @@ def __init__(self, method): super(http_post, self).__init__(method) def boundary(self): - return b''.join([chr(random.choice(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")) for i in range(32)]) + return to_bytes(''.join([random.choice("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") for i in range(32)])) def client_encode(self, buf): if self.has_sent_header: From 0e045a36e7194ff0971a16467add691788077b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Fri, 16 Jun 2017 21:51:45 +0800 Subject: [PATCH 461/487] fix http_post under py3 --- shadowsocks/obfsplugin/http_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/http_simple.py b/shadowsocks/obfsplugin/http_simple.py index 30d7ca91..6f1a05e4 100644 --- a/shadowsocks/obfsplugin/http_simple.py +++ b/shadowsocks/obfsplugin/http_simple.py @@ -250,7 +250,7 @@ def client_encode(self, buf): http_head += b"User-Agent: " + random.choice(self.user_agent) + b"\r\n" http_head += b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\nAccept-Encoding: gzip, deflate\r\n" http_head += b"Content-Type: multipart/form-data; boundary=" + self.boundary() + b"\r\nDNT: 1\r\n" - http_head += "Connection: keep-alive\r\n\r\n" + http_head += b"Connection: keep-alive\r\n\r\n" self.has_sent_header = True return http_head + buf From f6752fb21b3f9ea6475ca2a5ef5c369a4ce2e6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sat, 17 Jun 2017 00:27:09 +0800 Subject: [PATCH 462/487] refine Dockerfile --- Dockerfile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index a35c1fb5..a0ac0538 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,15 @@ FROM alpine -ENV SERVER_ADDR 0.0.0.0 -ENV SERVER_PORT 51348 -ENV PASSWORD psw -ENV METHOD aes-128-ctr -ENV PROTOCOL auth_aes128_md5 +ENV SERVER_ADDR 0.0.0.0 +ENV SERVER_PORT 51348 +ENV PASSWORD psw +ENV METHOD aes-128-ctr +ENV PROTOCOL auth_aes128_md5 ENV PROTOCOLPARAM 32 -ENV OBFS tls1.2_ticket_auth_compatible -ENV TIMEOUT 300 -ENV DNS_ADDR 8.8.8.8 -ENV DNS_ADDR_2 8.8.4.4 +ENV OBFS tls1.2_ticket_auth_compatible +ENV TIMEOUT 300 +ENV DNS_ADDR 8.8.8.8 +ENV DNS_ADDR_2 8.8.4.4 RUN apk update \ @@ -30,4 +30,5 @@ RUN wget --no-check-certificate https://github.com/shadowsocksr/shadowsocksr/arc WORKDIR ~/shadowsocksr/shadowsocks +EXPOSE 51348 CMD python ~/shadowsocksr/shadowsocks/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS -G $PROTOCOLPARAM From 19c868242a66d62de167b45e6f4da31f8ca431eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 18 Jun 2017 15:43:46 +0800 Subject: [PATCH 463/487] update mujson_mgr.py usage info --- mujson_mgr.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mujson_mgr.py b/mujson_mgr.py index 47e91a6b..2eb05d59 100644 --- a/mujson_mgr.py +++ b/mujson_mgr.py @@ -214,11 +214,11 @@ def print_server_help(): print('''usage: python mujson_manage.py -a|-d|-e|-c|-l [OPTION]... Actions: - -a ADD add/edit a user - -d DELETE delete a user - -e EDIT edit a user - -c CLEAR set u/d to zero - -l LIST display a user infomation or all users infomation + -a add/edit a user + -d delete a user + -e edit a user + -c set u&d to zero + -l display a user infomation or all users infomation Options: -u USER the user name @@ -232,6 +232,8 @@ def print_server_help(): -t TRANSFER max transfer for G bytes, default: 8388608 (8 PB or 8192 TB) -f FORBID set forbidden ports. Example (ban 1~79 and 81~100): -f "1-79,81-100" -i MUID set sub id to display (only work with -l) + -s SPEED set speed_limit_per_con + -S SPEED set speed_limit_per_user General options: -h, --help show this help message and exit From d8d19a99fcfb7a221fa302b52e816b8bf387006e Mon Sep 17 00:00:00 2001 From: muicoder Date: Mon, 19 Jun 2017 23:07:37 +0800 Subject: [PATCH 464/487] better --- Dockerfile | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0ac0538..47cf8889 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:3.6 ENV SERVER_ADDR 0.0.0.0 ENV SERVER_PORT 51348 @@ -11,24 +11,20 @@ ENV TIMEOUT 300 ENV DNS_ADDR 8.8.8.8 ENV DNS_ADDR_2 8.8.4.4 +ARG BRANCH=manyuser +ARG WORK=/root -RUN apk update \ - && apk add python \ - libsodium \ - unzip \ - wget \ - && rm -rf /var/cache/apk/* +RUN apk --no-cache add python \ + libsodium \ + wget -RUN wget --no-check-certificate https://github.com/shadowsocksr/shadowsocksr/archive/manyuser.zip -O /tmp/manyuser.zip \ - && unzip -d /tmp /tmp/manyuser.zip \ - && mv /tmp/shadowsocksr-manyuser ~/shadowsocksr \ - && rm -rf /tmp/* +RUN wget -qO- --no-check-certificate https://github.com/shadowsocksr/shadowsocksr/archive/$BRANCH.tar.gz | tar -xzf - -C $WORK -WORKDIR ~/shadowsocksr/shadowsocks +WORKDIR $WORK/shadowsocksr-$BRANCH/shadowsocks -EXPOSE 51348 -CMD python ~/shadowsocksr/shadowsocks/server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS -G $PROTOCOLPARAM +EXPOSE $SERVER_PORT +CMD python server.py -p $SERVER_PORT -k $PASSWORD -m $METHOD -O $PROTOCOL -o $OBFS -G $PROTOCOLPARAM From 7c6c671a5ebea4fbbd2bcaf2d14d9b91b9f65499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 22 Jun 2017 10:25:57 +0800 Subject: [PATCH 465/487] default work at "~" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 47cf8889..38622478 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV DNS_ADDR 8.8.8.8 ENV DNS_ADDR_2 8.8.4.4 ARG BRANCH=manyuser -ARG WORK=/root +ARG WORK=~ RUN apk --no-cache add python \ From adbaafa18a547dccec0eef205febf86391cb191a Mon Sep 17 00:00:00 2001 From: muicoder Date: Thu, 22 Jun 2017 12:32:55 +0800 Subject: [PATCH 466/487] fix default work --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38622478..ec7b2c6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ RUN apk --no-cache add python \ wget -RUN wget -qO- --no-check-certificate https://github.com/shadowsocksr/shadowsocksr/archive/$BRANCH.tar.gz | tar -xzf - -C $WORK +RUN mkdir -p $WORK && \ + wget -qO- --no-check-certificate https://github.com/shadowsocksr/shadowsocksr/archive/$BRANCH.tar.gz | tar -xzf - -C $WORK WORKDIR $WORK/shadowsocksr-$BRANCH/shadowsocks From 727d218ff2d17b711698ec5585a8b2db4c9413ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 22 Jun 2017 19:29:34 +0800 Subject: [PATCH 467/487] fix var name --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 7aae7e46..ca431e66 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -1294,9 +1294,9 @@ def add_user(self, uid, cfg): # user: binstr[4], passwd: str self._speed_tester_d[uid] = SpeedTester(speed) def del_user(self, uid): - if user in self.server_users: + if uid in self.server_users: del self.server_users[uid] - if user in self.server_users_cfg: + if uid in self.server_users_cfg: del self.server_users_cfg[uid] def add_transfer_u(self, user, transfer): From 57ba4f5f6749c317ea5daffe427fef0bee8a6625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 28 Jun 2017 15:54:14 +0800 Subject: [PATCH 468/487] add initmudbjson.sh --- initmudbjson.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 initmudbjson.sh diff --git a/initmudbjson.sh b/initmudbjson.sh new file mode 100755 index 00000000..213410a3 --- /dev/null +++ b/initmudbjson.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +bash initcfg.sh +sed -i 's/sspanelv2/mudbjson/g' userapiconfig.py +ip_addr=`ifconfig -a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\.168\..[0-9]\+\.[0-9]\+"|grep -v -e "10\.[0-9]\+\.[0-9]\+\.[0-9]\+"|awk '{print $2}'|tr -d "addr:"` +ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` +if [[ $ip_count == 1 ]]; then + echo "server IP is "${ip_addr} + sed -i 's/127\.0\.0\.1/'${ip_addr}'/g' userapiconfig.py + port=`python -c 'import random;print(random.randint(10000, 65536))'` + python mujson_mgr.py -a -p ${port} +else + echo "unable to detect server IP" +fi + From e21f01755af3721a2ed23c25f8865421e51e995c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 28 Jun 2017 16:05:43 +0800 Subject: [PATCH 469/487] update initmudbjson.sh --- initmudbjson.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/initmudbjson.sh b/initmudbjson.sh index 213410a3..b1dee3f2 100755 --- a/initmudbjson.sh +++ b/initmudbjson.sh @@ -1,12 +1,12 @@ #!/bin/bash bash initcfg.sh -sed -i 's/sspanelv2/mudbjson/g' userapiconfig.py +sed -i "s/API_INTERFACE = .\+\?\#/API_INTERFACE = \'mudbjson\' \#/g" userapiconfig.py ip_addr=`ifconfig -a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\.168\..[0-9]\+\.[0-9]\+"|grep -v -e "10\.[0-9]\+\.[0-9]\+\.[0-9]\+"|awk '{print $2}'|tr -d "addr:"` ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` if [[ $ip_count == 1 ]]; then echo "server IP is "${ip_addr} - sed -i 's/127\.0\.0\.1/'${ip_addr}'/g' userapiconfig.py + sed -i 's/SERVER_PUB_ADDR = .\+/SERVER_PUB_ADDR = '${ip_addr}'/g' userapiconfig.py port=`python -c 'import random;print(random.randint(10000, 65536))'` python mujson_mgr.py -a -p ${port} else From ab8765191eb68a5066c50b49bc4f298eab7a6a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 28 Jun 2017 16:12:14 +0800 Subject: [PATCH 470/487] update initmudbjson.sh --- initmudbjson.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/initmudbjson.sh b/initmudbjson.sh index b1dee3f2..c5f1e3ea 100755 --- a/initmudbjson.sh +++ b/initmudbjson.sh @@ -7,8 +7,11 @@ ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` if [[ $ip_count == 1 ]]; then echo "server IP is "${ip_addr} sed -i 's/SERVER_PUB_ADDR = .\+/SERVER_PUB_ADDR = '${ip_addr}'/g' userapiconfig.py - port=`python -c 'import random;print(random.randint(10000, 65536))'` - python mujson_mgr.py -a -p ${port} + user_count=`python mujson_mgr.py -l|grep -c ""` + if [[ $user_count == 0 ]]; then + port=`python -c 'import random;print(random.randint(10000, 65536))'` + python mujson_mgr.py -a -p ${port} + fi else echo "unable to detect server IP" fi From 1a2c7ed933452e8b05ead7fa8b39e3dfb15c53f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 28 Jun 2017 16:25:21 +0800 Subject: [PATCH 471/487] update initmudbjson.sh --- initmudbjson.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/initmudbjson.sh b/initmudbjson.sh index c5f1e3ea..df4be7a8 100755 --- a/initmudbjson.sh +++ b/initmudbjson.sh @@ -7,7 +7,7 @@ ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` if [[ $ip_count == 1 ]]; then echo "server IP is "${ip_addr} sed -i 's/SERVER_PUB_ADDR = .\+/SERVER_PUB_ADDR = '${ip_addr}'/g' userapiconfig.py - user_count=`python mujson_mgr.py -l|grep -c ""` + user_count=`python mujson_mgr.py -l|grep -c -e "[0-9]"` if [[ $user_count == 0 ]]; then port=`python -c 'import random;print(random.randint(10000, 65536))'` python mujson_mgr.py -a -p ${port} From bc930031a9e6d429e23cc8e5801d8206312cc043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 28 Jun 2017 16:47:05 +0800 Subject: [PATCH 472/487] update initmudbjson.sh --- initmudbjson.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/initmudbjson.sh b/initmudbjson.sh index df4be7a8..30883318 100755 --- a/initmudbjson.sh +++ b/initmudbjson.sh @@ -6,7 +6,7 @@ ip_addr=`ifconfig -a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\ ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` if [[ $ip_count == 1 ]]; then echo "server IP is "${ip_addr} - sed -i 's/SERVER_PUB_ADDR = .\+/SERVER_PUB_ADDR = '${ip_addr}'/g' userapiconfig.py + sed -i "s/SERVER_PUB_ADDR = .\+/SERVER_PUB_ADDR = \'"${ip_addr}"\'/g" userapiconfig.py user_count=`python mujson_mgr.py -l|grep -c -e "[0-9]"` if [[ $user_count == 0 ]]; then port=`python -c 'import random;print(random.randint(10000, 65536))'` From a629d1baeca52a3c0190f6b0efbfc67212f20b95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 28 Jun 2017 20:10:15 +0800 Subject: [PATCH 473/487] update initmudbjson.sh --- initmudbjson.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/initmudbjson.sh b/initmudbjson.sh index 30883318..5ef4acc1 100755 --- a/initmudbjson.sh +++ b/initmudbjson.sh @@ -4,6 +4,12 @@ bash initcfg.sh sed -i "s/API_INTERFACE = .\+\?\#/API_INTERFACE = \'mudbjson\' \#/g" userapiconfig.py ip_addr=`ifconfig -a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\.168\..[0-9]\+\.[0-9]\+"|grep -v -e "10\.[0-9]\+\.[0-9]\+\.[0-9]\+"|awk '{print $2}'|tr -d "addr:"` ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` + +if [[ $ip_count == 1 ]]; then + ip_addr=`ip a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\.168\..[0-9]\+\.[0-9]\+"|grep -v -e "10\.[0-9]\+\.[0-9]\+\.[0-9]\+"|awk '{print $2}'"` + ip_addr=${ip_addr%/*} + ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` +fi if [[ $ip_count == 1 ]]; then echo "server IP is "${ip_addr} sed -i "s/SERVER_PUB_ADDR = .\+/SERVER_PUB_ADDR = \'"${ip_addr}"\'/g" userapiconfig.py From 339c1ccee25b1b2157924324b22374c6514a4313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Mon, 3 Jul 2017 14:49:10 +0800 Subject: [PATCH 474/487] fix connections ref --- shadowsocks/tcprelay.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index ca431e66..506e1cee 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -138,6 +138,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._remote_udp = False self._config = config self._dns_resolver = dns_resolver + self._add_ref = 0 if not self._create_encryptor(config): return @@ -217,6 +218,7 @@ def __init__(self, server, fd_to_handlers, loop, local_sock, config, self._update_activity() self._server.add_connection(1) self._server.stat_add(self._client_address[0], 1) + self._add_ref = 1 self.speed_tester_u = SpeedTester(config.get("speed_limit_per_con", 0)) self.speed_tester_d = SpeedTester(config.get("speed_limit_per_con", 0)) self._recv_u_max_size = BUF_SIZE @@ -1159,8 +1161,9 @@ def destroy(self): self._encryptor = None self._dns_resolver.remove_callback(self._handle_dns_resolved) self._server.remove_handler(self) - self._server.add_connection(-1) - self._server.stat_add(self._client_address[0], -1) + if self._add_ref > 0: + self._server.add_connection(-1) + self._server.stat_add(self._client_address[0], -1) class TCPRelay(object): def __init__(self, config, dns_resolver, is_local, stat_callback=None, stat_counter=None): From 0c1a9368c907d6e0ccd0f7d67d2c1036e33e2cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 12 Jul 2017 22:28:02 +0800 Subject: [PATCH 475/487] fix initmudbjson.sh, remove LRUCache init log --- initmudbjson.sh | 2 +- shadowsocks/lru_cache.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/initmudbjson.sh b/initmudbjson.sh index 5ef4acc1..09b07f33 100755 --- a/initmudbjson.sh +++ b/initmudbjson.sh @@ -6,7 +6,7 @@ ip_addr=`ifconfig -a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\ ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` if [[ $ip_count == 1 ]]; then - ip_addr=`ip a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\.168\..[0-9]\+\.[0-9]\+"|grep -v -e "10\.[0-9]\+\.[0-9]\+\.[0-9]\+"|awk '{print $2}'"` + ip_addr=`ip a|grep inet|grep -v inet6|grep -v "127.0.0."|grep -v -e "192\.168\..[0-9]\+\.[0-9]\+"|grep -v -e "10\.[0-9]\+\.[0-9]\+\.[0-9]\+"|awk '{print $2}'` ip_addr=${ip_addr%/*} ip_count=`echo $ip_addr|grep -e "^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$" -c` fi diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 328ef85a..ab0d2108 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -29,7 +29,6 @@ try: from collections import OrderedDict - print("loaded collections.OrderedDict") except: from shadowsocks.ordereddict import OrderedDict From 4198ec84cecb043eca3a9ffcb5c2a1aae94990ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Sun, 16 Jul 2017 11:56:17 +0800 Subject: [PATCH 476/487] not filter mu port --- db_transfer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/db_transfer.py b/db_transfer.py index 3a1441d1..67bda608 100644 --- a/db_transfer.py +++ b/db_transfer.py @@ -122,12 +122,14 @@ def del_server_out_of_bound_safe(self, last_rows, rows): logging.error('more than one user use the same port [%s]' % (port,)) continue + if 'protocol' in cfg and 'protocol_param' in cfg and common.to_str(cfg['protocol']) in obfs.mu_protocol(): + if '#' in common.to_str(cfg['protocol_param']): + mu_servers[port] = passwd + allow = True + if allow: - allow_users[port] = cfg - if 'protocol' in cfg and 'protocol_param' in cfg and common.to_str(cfg['protocol']) in obfs.mu_protocol(): - if '#' in common.to_str(cfg['protocol_param']): - mu_servers[port] = passwd - del allow_users[port] + if port not in mu_servers: + allow_users[port] = cfg cfgchange = False if port in ServerPool.get_instance().tcp_servers_pool: From ab236ff03ef102e00eae7bc6a4f6a081a5eca31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Tue, 25 Jul 2017 17:15:49 +0800 Subject: [PATCH 477/487] fix compatible of tls1.2_ticket_fastauth --- shadowsocks/obfsplugin/obfs_tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 19471fc0..73199ab1 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -194,7 +194,7 @@ def server_encode(self, buf): def decode_error_return(self, buf): self.handshake_status = -1 self.overhead = 0 - if self.method == 'tls1.2_ticket_auth': + if self.method in ['tls1.2_ticket_auth', 'tls1.2_ticket_fastauth']: return (b'E'*2048, False, False) return (buf, True, False) From 38ef6a25467a0ed8bad1e4b43831a6dcf0cc8faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 26 Jul 2017 07:47:28 +0800 Subject: [PATCH 478/487] add auth_chain_b --- shadowsocks/obfsplugin/auth_chain.py | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index 5eed68cd..c0ced3e1 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -31,6 +31,7 @@ import zlib import hmac import hashlib +import bisect import shadowsocks from shadowsocks import common, lru_cache, encrypt @@ -40,9 +41,12 @@ def create_auth_chain_a(method): return auth_chain_a(method) +def create_auth_chain_b(method): + return auth_chain_b(method) obfs_map = { 'auth_chain_a': (create_auth_chain_a,), + 'auth_chain_b': (create_auth_chain_b,), } class xorshift128plus(object): @@ -630,3 +634,46 @@ def server_udp_post_decrypt(self, buf): def dispose(self): self.server_info.data.remove(self.user_id, self.client_id) +class auth_chain_b(auth_chain_a): + def __init__(self, method): + super(auth_chain_b, self).__init__(method) + self.salt = b"auth_chain_b" + self.no_compatible_method = 'auth_chain_b' + self.data_size_list = [] + + def init_data_size(self, key): + if self.data_size_list: + self.data_size_list = [] + random = xorshift128plus() + random.init_from_bin(key) + list_len = random.next() % 32 + 8; + for i in range(0, list_len): + self.data_size_list.append((int)(random.next() % 1440)) + self.data_size_list.sort() + + def set_server_info(self, server_info): + self.server_info = server_info + try: + max_client = int(server_info.protocol_param.split('#')[0]) + except: + max_client = 64 + self.server_info.data.set_max_client(max_client) + self.init_data_size(self.server_info.key) + + def rnd_data_len(self, buf_size, last_hash, random): + if buf_size >= 1440: + return 0 + random.init_from_bin_len(last_hash, buf_size) + pos = bisect.bisect_left(self.data_size_list, buf_size) + final_pos = pos + random.next() % (len(self.data_size_list) + 1 - pos) + if final_pos < len(self.data_size_list): + return self.data_size_list[final_pos] - buf_size + + if buf_size > 1300: + return random.next() % 31 + if buf_size > 900: + return random.next() % 127 + if buf_size > 400: + return random.next() % 521 + return random.next() % 1021 + From 838caa3f1d6432b0e20e7af330fbe97a8a511127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 26 Jul 2017 09:28:13 +0800 Subject: [PATCH 479/487] fix overhead --- shadowsocks/obfsplugin/auth_chain.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index c0ced3e1..b39624f2 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -646,7 +646,7 @@ def init_data_size(self, key): self.data_size_list = [] random = xorshift128plus() random.init_from_bin(key) - list_len = random.next() % 32 + 8; + list_len = random.next() % 16 + 4; for i in range(0, list_len): self.data_size_list.append((int)(random.next() % 1440)) self.data_size_list.sort() @@ -664,10 +664,10 @@ def rnd_data_len(self, buf_size, last_hash, random): if buf_size >= 1440: return 0 random.init_from_bin_len(last_hash, buf_size) - pos = bisect.bisect_left(self.data_size_list, buf_size) + pos = bisect.bisect_left(self.data_size_list, buf_size + self.server_info.overhead) final_pos = pos + random.next() % (len(self.data_size_list) + 1 - pos) if final_pos < len(self.data_size_list): - return self.data_size_list[final_pos] - buf_size + return self.data_size_list[final_pos] - buf_size - self.server_info.overhead if buf_size > 1300: return random.next() % 31 From 7dddf07da49f5c245caa0bfd0fc58615bc4815cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 26 Jul 2017 10:20:17 +0800 Subject: [PATCH 480/487] fix overhead --- shadowsocks/obfsplugin/obfs_tls.py | 2 ++ shadowsocks/tcprelay.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/shadowsocks/obfsplugin/obfs_tls.py b/shadowsocks/obfsplugin/obfs_tls.py index 73199ab1..7f1f2336 100644 --- a/shadowsocks/obfsplugin/obfs_tls.py +++ b/shadowsocks/obfsplugin/obfs_tls.py @@ -193,6 +193,8 @@ def server_encode(self, buf): def decode_error_return(self, buf): self.handshake_status = -1 + if self.overhead > 0: + self.server_info.overhead -= self.overhead self.overhead = 0 if self.method in ['tls1.2_ticket_auth', 'tls1.2_ticket_fastauth']: return (b'E'*2048, False, False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 506e1cee..78b32575 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -858,6 +858,10 @@ def _on_local_read(self): if self._encrypt_correct: try: obfs_decode = self._obfs.server_decode(data) + if self._stage == STAGE_INIT: + self._overhead = self._obfs.get_overhead(self._is_local) + self._protocol.get_overhead(self._is_local) + server_info = self._protocol.get_server_info() + server_info.overhead = self._overhead except Exception as e: shell.print_exception(e) logging.error("exception from %s:%d" % (self._client_address[0], self._client_address[1])) From cb1dcacd4b4f200648de059d44d25e2d454b0389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 26 Jul 2017 12:13:26 +0800 Subject: [PATCH 481/487] refine recv size try more data_size_list --- shadowsocks/obfsplugin/auth_chain.py | 13 ++++++++++++- shadowsocks/tcprelay.py | 6 +++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index b39624f2..39490082 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -640,16 +640,22 @@ def __init__(self, method): self.salt = b"auth_chain_b" self.no_compatible_method = 'auth_chain_b' self.data_size_list = [] + self.data_size_list2 = [] def init_data_size(self, key): if self.data_size_list: self.data_size_list = [] + self.data_size_list2 = [] random = xorshift128plus() random.init_from_bin(key) - list_len = random.next() % 16 + 4; + list_len = random.next() % 16 + 4 for i in range(0, list_len): self.data_size_list.append((int)(random.next() % 1440)) self.data_size_list.sort() + list_len = random.next() % 32 + 32 + for i in range(0, list_len): + self.data_size_list2.append((int)(random.next() % 1440)) + self.data_size_list2.sort() def set_server_info(self, server_info): self.server_info = server_info @@ -669,6 +675,11 @@ def rnd_data_len(self, buf_size, last_hash, random): if final_pos < len(self.data_size_list): return self.data_size_list[final_pos] - buf_size - self.server_info.overhead + pos = bisect.bisect_left(self.data_size_list2, buf_size + self.server_info.overhead) + final_pos = pos + random.next() % (len(self.data_size_list2) + 1 - pos) + if final_pos < len(self.data_size_list2): + return self.data_size_list2[final_pos] - buf_size - self.server_info.overhead + if buf_size > 1300: return random.next() % 31 if buf_size > 900: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 78b32575..595e2be7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -816,15 +816,15 @@ def _get_read_size(self, sock, recv_buffer_size, up): if self._overhead == 0: return recv_buffer_size buffer_size = len(sock.recv(recv_buffer_size, socket.MSG_PEEK)) + frame_size = self._tcp_mss - self._overhead if up: buffer_size = min(buffer_size, self._recv_u_max_size) - self._recv_u_max_size = min(self._recv_u_max_size + self._tcp_mss - self._overhead, BUF_SIZE) + self._recv_u_max_size = min(self._recv_u_max_size + frame_size, BUF_SIZE) else: buffer_size = min(buffer_size, self._recv_d_max_size) - self._recv_d_max_size = min(self._recv_d_max_size + self._tcp_mss - self._overhead, BUF_SIZE) + self._recv_d_max_size = min(self._recv_d_max_size + frame_size, BUF_SIZE) if buffer_size == recv_buffer_size: return buffer_size - frame_size = self._tcp_mss - self._overhead if buffer_size > frame_size: buffer_size = int(buffer_size / frame_size) * frame_size return buffer_size From 604cbf8a78c8ebe7a0c990bced9cceb1614d5952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Wed, 26 Jul 2017 14:47:30 +0800 Subject: [PATCH 482/487] more small packet --- shadowsocks/obfsplugin/auth_chain.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index 39490082..d748ef03 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -648,13 +648,13 @@ def init_data_size(self, key): self.data_size_list2 = [] random = xorshift128plus() random.init_from_bin(key) - list_len = random.next() % 16 + 4 + list_len = random.next() % 8 + 4 for i in range(0, list_len): - self.data_size_list.append((int)(random.next() % 1440)) + self.data_size_list.append((int)(random.next() % 2340 % 2040 % 1440)) self.data_size_list.sort() - list_len = random.next() % 32 + 32 + list_len = random.next() % 16 + 16 for i in range(0, list_len): - self.data_size_list2.append((int)(random.next() % 1440)) + self.data_size_list2.append((int)(random.next() % 2340 % 2040 % 1440)) self.data_size_list2.sort() def set_server_info(self, server_info): From e8b4bb05a0799d0af973a1e7846fae0df3b283ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 27 Jul 2017 13:25:57 +0800 Subject: [PATCH 483/487] adjust --- shadowsocks/obfsplugin/auth_chain.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index d748ef03..67a2a7e6 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -652,7 +652,7 @@ def init_data_size(self, key): for i in range(0, list_len): self.data_size_list.append((int)(random.next() % 2340 % 2040 % 1440)) self.data_size_list.sort() - list_len = random.next() % 16 + 16 + list_len = random.next() % 16 + 8 for i in range(0, list_len): self.data_size_list2.append((int)(random.next() % 2340 % 2040 % 1440)) self.data_size_list2.sort() @@ -671,12 +671,12 @@ def rnd_data_len(self, buf_size, last_hash, random): return 0 random.init_from_bin_len(last_hash, buf_size) pos = bisect.bisect_left(self.data_size_list, buf_size + self.server_info.overhead) - final_pos = pos + random.next() % (len(self.data_size_list) + 1 - pos) + final_pos = pos + random.next() % (len(self.data_size_list) * 2) if final_pos < len(self.data_size_list): return self.data_size_list[final_pos] - buf_size - self.server_info.overhead pos = bisect.bisect_left(self.data_size_list2, buf_size + self.server_info.overhead) - final_pos = pos + random.next() % (len(self.data_size_list2) + 1 - pos) + final_pos = pos + random.next() % (len(self.data_size_list2)) if final_pos < len(self.data_size_list2): return self.data_size_list2[final_pos] - buf_size - self.server_info.overhead From e3ec6bb5f81a8672db21ddd69bcc77bb26a81a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 27 Jul 2017 14:20:57 +0800 Subject: [PATCH 484/487] adjust --- shadowsocks/obfsplugin/auth_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index 67a2a7e6..8e7d5c7f 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -671,7 +671,7 @@ def rnd_data_len(self, buf_size, last_hash, random): return 0 random.init_from_bin_len(last_hash, buf_size) pos = bisect.bisect_left(self.data_size_list, buf_size + self.server_info.overhead) - final_pos = pos + random.next() % (len(self.data_size_list) * 2) + final_pos = pos + random.next() % (len(self.data_size_list) + int(len(self.data_size_list) / 2)) if final_pos < len(self.data_size_list): return self.data_size_list[final_pos] - buf_size - self.server_info.overhead From b3cf97c44e0b7023354b961c0e447470b53e1f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 27 Jul 2017 14:59:49 +0800 Subject: [PATCH 485/487] adjust --- shadowsocks/obfsplugin/auth_chain.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shadowsocks/obfsplugin/auth_chain.py b/shadowsocks/obfsplugin/auth_chain.py index 8e7d5c7f..26097bfb 100644 --- a/shadowsocks/obfsplugin/auth_chain.py +++ b/shadowsocks/obfsplugin/auth_chain.py @@ -671,7 +671,7 @@ def rnd_data_len(self, buf_size, last_hash, random): return 0 random.init_from_bin_len(last_hash, buf_size) pos = bisect.bisect_left(self.data_size_list, buf_size + self.server_info.overhead) - final_pos = pos + random.next() % (len(self.data_size_list) + int(len(self.data_size_list) / 2)) + final_pos = pos + random.next() % (len(self.data_size_list)) if final_pos < len(self.data_size_list): return self.data_size_list[final_pos] - buf_size - self.server_info.overhead @@ -679,6 +679,8 @@ def rnd_data_len(self, buf_size, last_hash, random): final_pos = pos + random.next() % (len(self.data_size_list2)) if final_pos < len(self.data_size_list2): return self.data_size_list2[final_pos] - buf_size - self.server_info.overhead + if final_pos < pos + len(self.data_size_list2) - 1: + return 0 if buf_size > 1300: return random.next() % 31 From fd723a92c488d202b407323f0512987346944136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A0=B4=E5=A8=83=E9=85=B1?= Date: Thu, 27 Jul 2017 15:50:44 +0800 Subject: [PATCH 486/487] bump --- CHANGES | 6 ++++++ shadowsocks/version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d5fe72fb..0cd91768 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +3.4.0 2017-07-27 +- add auth_chain_b +- add initmudbjson.sh +- allow set speed limit in runtime +- fix bugs & mem leak + 3.3.3 2017-06-03 - add DNS cache - add tls1.2_ticket_fastauth diff --git a/shadowsocks/version.py b/shadowsocks/version.py index 74eac22b..f3e1ef79 100644 --- a/shadowsocks/version.py +++ b/shadowsocks/version.py @@ -16,5 +16,5 @@ # under the License. def version(): - return '3.3.3 2017-06-03' + return '3.4.0 2017-07-27' From 97f71a536c66e0a664e77e36950d77b1176b7f3d Mon Sep 17 00:00:00 2001 From: KeJun Date: Fri, 18 Aug 2017 11:27:44 +0800 Subject: [PATCH 487/487] fix Docker file wget url --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ec7b2c6e..c23b6eb0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN apk --no-cache add python \ RUN mkdir -p $WORK && \ - wget -qO- --no-check-certificate https://github.com/shadowsocksr/shadowsocksr/archive/$BRANCH.tar.gz | tar -xzf - -C $WORK + wget -qO- --no-check-certificate https://github.com/shadowsocksr-backup/shadowsocksr/archive/$BRANCH.tar.gz | tar -xzf - -C $WORK WORKDIR $WORK/shadowsocksr-$BRANCH/shadowsocks