The branch, build-main has been updated
via 15fba676bee4d4cebb9d44afe30259ca282ea7ef (commit)
via 42032bd1bbcef0555bb15baf022704c09d94c342 (commit)
via 26629de904cb2dc49f576d156efd6913ab7cccdd (commit)
via 9ffb7417721a1ca2ab19e48cca2a545c92a56e0e (commit)
via 02b425048b6beca03c0ae202d54005157dc71d5f (commit)
via 38e1f3976082c99519843ab8f4cfa0730bdd818c (commit)
via 64282631fe19aa8bea452a059edbff1d36b8d425 (commit)
via 7f25682383416a9b0487d5cc0268dae9a8bdd0b4 (commit)
via 8e73d26ce1bc5b1f8f1ce314877f90b37e6e8898 (commit)
via fe61669b7ab11de66223f62c52f380fc2b1e43bf (commit)
via e48dab769735322633d28a54c19f25bdeed27aa3 (commit)
via 09dff9341cb3dc697bce90864fe3e9c76f059269 (commit)
via 2573ffdec27908663f7ae786e6e9af4230dc69a8 (commit)
via 9557b67570875897d1f45f61f174461c45b1a7c6 (commit)
via d4054a17444c93618020c4412e8b49361371f3cf (commit)
via 140299b062adce94b65dc10ac19bb97497af87cc (commit)
via 3f3e1d0961d98a8a4b993efc0ed06b0295f6bcac (commit)
via 3f866e07a2c228fd3681a011f49794b73b51692a (commit)
from 8676299cde99da2c19307431c2a81f46b16b31dc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
-----------------------------------------------------------------------
Summary of changes:
debian/changelog | 17 +++++
debian/control | 2 +
sbin/x2gobroker-pubkeyauthorizer | 2 +-
x2gobroker/__init__.py | 11 ++-
x2gobroker/agent.py | 6 +-
x2gobroker/brokers/base_broker.py | 106 ++++++++++++++++++++++++----
x2gobroker/tests/test_utils.py | 102 ++++++++++++++++++++++++++
x2gobroker/tests/test_web_plain_inifile.py | 6 +-
x2gobroker/utils.py | 45 ++++++++++++
x2gobroker/web/extras.py | 2 +-
x2gobroker/web/html.py | 2 +-
x2gobroker/web/plain.py | 2 +-
12 files changed, 280 insertions(+), 23 deletions(-)
create mode 100644 x2gobroker/tests/test_utils.py
The diff of changes is:
diff --git a/debian/changelog b/debian/changelog
index 872f3aa..7b5e8b0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,20 @@
+x2gobroker (0.0.0.7-0~x2go1) unstable; urgency=low
+
+ [ Mike Gabriel ]
+ * New upstream version (0.0.0.7):
+ - Add algorithm to ,,normalize'' hostnames used in session profiles
+ vs. those returned by the broker agent. (Fixes: #133).
+ - Ignore off-line X2Go servers in multi-node load-balanced setups.
+ (Fixes: #132).
+ - Return some sane output to x2goclient if the / all configured X2Go
+ server(s) is/are down.
+ - Tornado: Use RequestHandler.set_header() instead of
+ RequestHandler.add_header().
+ * /debian/control:
+ + Build-Depend on python-paste, python-nose (testsuite needs them).
+
+ -- Mike Gabriel <mike.gabriel(a)das-netzwerkteam.de> Thu, 07 Mar 2013 07:14:31 +0100
+
x2gobroker (0.0.0.6-0~x2go1) unstable; urgency=low
[ Mike Gabriel ]
diff --git a/debian/control b/debian/control
index ece5439..79dc5cb 100644
--- a/debian/control
+++ b/debian/control
@@ -11,6 +11,8 @@ Build-Depends:
dpkg-dev (>= 1.16.1~),
python (>= 2.6.5-0~),
python-setuptools,
+ python-nose,
+ python-paste,
Standards-Version: 3.9.3
XS-Python-Version: >= 2.4
diff --git a/sbin/x2gobroker-pubkeyauthorizer b/sbin/x2gobroker-pubkeyauthorizer
index c1e9f41..39440bb 100755
--- a/sbin/x2gobroker-pubkeyauthorizer
+++ b/sbin/x2gobroker-pubkeyauthorizer
@@ -36,7 +36,7 @@ import logging.config
from pwd import getpwnam
from grp import getgrnam
-__VERSION__ = '0.0.0.6'
+__VERSION__ = '0.0.0.7'
__AUTHOR__ = 'Mike Gabriel (X2Go Project) <mike.gabriel(a)das-netzwerkteam.de>'
PROG_NAME = os.path.basename(sys.argv[0])
diff --git a/x2gobroker/__init__.py b/x2gobroker/__init__.py
index 2c644a2..f63bf9c 100644
--- a/x2gobroker/__init__.py
+++ b/x2gobroker/__init__.py
@@ -18,5 +18,14 @@
# Free Software Foundation, Inc.,
# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
-__VERSION__ = '0.0.0.6'
+__VERSION__ = '0.0.0.7'
__AUTHOR__ = 'Mike Gabriel (X2Go Project) <mike.gabriel(a)das-netzwerkteam.de>'
+
+from loggers import logger_error
+
+class X2GoBrokerBaseException(BaseException):
+ def __init__(self, *args, **kwargs):
+ BaseException.__init__(self, *args, **kwargs)
+ logger_error.error('An exceptional problem occurred: {exception}("{msg}")'.format(exception=type(self).__name__, msg=str(self)))
+
+class X2GoBrokerAgentException(X2GoBrokerBaseException): pass
diff --git a/x2gobroker/agent.py b/x2gobroker/agent.py
index 9b63f00..001a800 100644
--- a/x2gobroker/agent.py
+++ b/x2gobroker/agent.py
@@ -24,6 +24,7 @@ import paramiko
import cStringIO
import time
import threading
+import socket
import x2gobroker._paramiko
x2gobroker._paramiko.monkey_patch_paramiko()
@@ -82,6 +83,7 @@ def call_local_broker_agent(username, mode, cmdline_args=[]):
if result[0].startswith('OK'):
return [ r for r in result[1:] if r ]
+ raise x2gobroker.X2GoBrokerAgentException('Query to local X2Go Broker Agent failed with no response')
def call_remote_broker_agent(username, mode, cmdline_args=[], remote_agent=None):
"""\
@@ -132,8 +134,8 @@ def call_remote_broker_agent(username, mode, cmdline_args=[], remote_agent=None)
client.close()
if result and result[0].startswith('OK'):
return [ r for r in result[1:] if r ]
- except paramiko.SSHException:
- logger_error.error('could not connect to remote X2Go Broker Agent (user: {user}, hostname: {hostname}, port: {port}'.format(user=remote_username, hostname=remote_hostname, port=remote_port))
+ except (paramiko.SSHException, paramiko.AuthenticationException, paramiko.BadHostKeyException, socket.error):
+ raise x2gobroker.X2GoBrokerAgentException('Query to remote X2Go Broker Agent (user: {user}, hostname: {hostname}, port: {port}) failed'.format(user=remote_username, hostname=remote_hostname, port=remote_port))
def list_sessions(username, query_mode='LOCAL', remote_agent=None):
diff --git a/x2gobroker/brokers/base_broker.py b/x2gobroker/brokers/base_broker.py
index 9001039..89aadac 100644
--- a/x2gobroker/brokers/base_broker.py
+++ b/x2gobroker/brokers/base_broker.py
@@ -787,6 +787,31 @@ class X2GoBroker(object):
return list_of_profiles
+ def random_remote_agent(self, profile_id, exclude_agents=[]):
+ """\
+ Randomly choose a remote agent for agent query.
+
+ @param profile_id: choose remote agent for this profile ID
+ @type profile_id: C{unicode}
+ @param exclude_agents: a list of remote agent dict objects to be exclude from the random choice
+ @type exclude_agents: C{list}
+
+ """
+ profile = self.get_profile(profile_id)
+ server_list = profile[u'host']
+
+ for agent in exclude_agents:
+ if agent['hostname'] in server_list:
+ server_list.remove(agent['hostname'])
+
+ remote_agent = None
+ if server_list:
+ random.shuffle(server_list)
+ remote_agent_server = server_list[0]
+ remote_agent_port = profile[u'sshport']
+ remote_agent = {u'hostname': remote_agent_server, u'port': remote_agent_port, }
+ return remote_agent
+
def select_session(self, profile_id, username=None):
"""\
Start/resume a session by selecting a profile name offered by the X2Go client.
@@ -804,30 +829,85 @@ class X2GoBroker(object):
# if we have more than one server, pick one server randomly for X2Go Broker Agent queries
server_list = profile[u'host']
+ random.shuffle(server_list)
agent_query_mode = self.get_agent_query_mode()
remote_agent = None
if agent_query_mode.upper() == u'SSH':
- random.shuffle(server_list)
- remote_agent_server = server_list[0]
- remote_agent_port = profile[u'sshport']
- remote_agent = {u'hostname': remote_agent_server, u'port': remote_agent_port, }
+ remote_agent = self.random_remote_agent(profile_id)
+ # detect best X2Go server for this user if load balancing is configured
if len(server_list) >= 2 and username:
- busy_servers = x2gobroker.agent.find_busy_servers(username=username, query_mode=agent_query_mode, remote_agent = remote_agent)
+ # query remote agent for session info, if one of the server's is down, we will try the next one...
+ busy_servers = None
+ exclude_agents = []
+ while busy_servers is None and remote_agent:
+
+ try:
+ busy_servers = x2gobroker.agent.find_busy_servers(username=username, query_mode=agent_query_mode, remote_agent=remote_agent)
+ except x2gobroker.X2GoBrokerAgentException:
+ logger_broker.warning('base_broker.X2GoBroker.select_session(): failed to query broker agent (quey-mode: {query_mode}, remote_agent: {remote_agent})'.format(query_mode=agent_query_mode, remote_agent=remote_agent))
+
+ if agent_query_mode.upper() == u'SSH':
+ # mark this agent as bad
+ exclude_agents.append(remote_agent)
+ # also remove this agent from the list of available servers as the machine is probably down
+ server_list.remove(remote_agent['hostname'])
+
+ remote_agent = self.random_remote_agent(profile_id, exclude_agents=exclude_agents)
+ else:
+ remote_agent = None
+
+ if busy_servers is not None:
+
+ # if we do not get until here, then all broker agent calls have failed!!!!
+
+ # when detecting the server load we have to support handling of differing subdomains (config
+ # file vs. server load returned by x2gobroker agent). Best approach: all members of a multi-node
+ # server farm either (a) do not have a subdomain in their hostname or (b) the subdomain is
+ # is identical.
+
+ # Example:
+ #
+ # ts01, ts02 - hostnames as returned by agent
+ # ts01.intern, ts02.intern - hostnames configured in session profile option ,,host''
+ # -> this will result in the subdomain .intern being stripped off from the hostnames before detecting
+ # the best server for this user
- for server in server_list:
- if server not in busy_servers.keys():
- busy_servers[server] = 0
+ ### NORMALIZE (=reduce to hostname only) X2Go server names (as found in config) if possible
+ server_list_normalized, subdomains_config = x2gobroker.utils.normalize_hostnames(server_list)
- busy_server_list = [ (load, server) for server, load in busy_servers.items() ]
- busy_server_list.sort()
+ ### NORMALIZE X2Go server names (as returned by x2gobroker agent)--only if the hostnames in the config share the same subdomain
+ if len(subdomains_config) == 1:
- best_server = busy_server_list[0][1]
+ busy_servers_normalized, subdomains_agent = x2gobroker.utils.normalize_hostnames(busy_servers)
+ if len(subdomains_agent) <= 1:
+
+ # all X2Go servers in the multi-node server farm are in the same DNS subdomain
+ # we can operate on hostname-only hostnames
+ server_list = server_list_normalized
+ busy_servers = busy_servers_normalized
+
+ for server in server_list:
+ if server not in busy_servers.keys():
+ busy_servers[server] = 0
+
+ busy_server_list = [ (load, server) for server, load in busy_servers.items() ]
+ busy_server_list.sort()
+
+ logger_broker.debug('base_broker.X2GoBroker.select_session(): load balancer analysis: {server_load}'.format(server_load=unicode(busy_server_list)))
+
+ best_server = busy_server_list[0][1]
+
+ else:
+ logger_broker.warning('base_broker.X2GoBroker.select_session(): all expected broker agents failed to respond, this does not look good. We tried these agent hosts: {agent_hosts}'.format(agent_hosts=unicode(server_list)))
+ if server_list: best_server = server_list[0]
+ else: return { 'server': 'no-server-available', 'port': profile[u'sshport'], }
else:
- best_server = server_list[0]
+ if server_list: best_server = server_list[0]
+ else: return { 'server': 'no-server-available', 'port': profile[u'sshport'], }
selected_session = {
'server': best_server,
@@ -877,7 +957,7 @@ class X2GoBroker(object):
query_mode=agent_query_mode,
remote_agent=remote_agent,
delay_deletion=20,
- ),
+ )
return selected_session
diff --git a/x2gobroker/tests/test_utils.py b/x2gobroker/tests/test_utils.py
new file mode 100644
index 0000000..871dca7
--- /dev/null
+++ b/x2gobroker/tests/test_utils.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012 by Mike Gabriel <mike.gabriel(a)das-netzwerkteam.de>
+#
+# X2Go Session Broker is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# X2Go Session Broker is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program; if not, write to the
+# Free Software Foundation, Inc.,
+# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+import unittest
+import tempfile
+
+# Python X2GoBroker modules
+import x2gobroker.utils
+
+class TestX2GoBrokerUtils(unittest.TestCase):
+
+ ### TEST FUNCTION: normalize_hostnames() with server lists (ListType)
+
+ def test_normalize_hostnames_listtype(self):
+
+ server_list = ['ts01', 'ts02', 'ts03',]
+ expected_server_list = ['ts01', 'ts02', 'ts03',]
+ server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_list)
+ server_list_n.sort()
+ self.assertEqual(expected_server_list, server_list_n)
+ self.assertEqual([], subdomains)
+
+ server_list = ['ts01.intern', 'ts02.intern', 'ts03.intern',]
+ expected_server_list = ['ts01', 'ts02', 'ts03',]
+ server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_list)
+ server_list_n.sort()
+ self.assertEqual(expected_server_list, server_list_n)
+ self.assertEqual(['intern'], subdomains)
+
+ server_list = ['ts01.intern', 'ts02.intern', 'ts03.extern',]
+ expected_server_list = ['ts01.intern', 'ts02.intern', 'ts03.extern',]
+ server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_list)
+ server_list_n.sort()
+ subdomains.sort()
+ self.assertEqual(expected_server_list, server_list_n)
+ self.assertEqual(['extern', 'intern'], subdomains)
+
+ server_tuple = ('ts01.intern', 'ts02.intern', 'ts03.extern',)
+ expected_server_list = ['ts01.intern', 'ts02.intern', 'ts03.extern',]
+ server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_tuple)
+ server_list_n.sort()
+ subdomains.sort()
+ self.assertEqual(expected_server_list, server_list_n)
+ self.assertEqual(['extern', 'intern'], subdomains)
+
+ ### TEST FUNCTION: normalize_hostnames() with server load (DictType)
+
+ def test_normalize_hostnames_dicttype(self):
+
+ server_load = {'ts01': 10, 'ts02': 20, 'ts03': 30,}
+ expected_server_load = {'ts01': 10, 'ts02': 20, 'ts03': 30,}
+ server_load_n, subdomains = x2gobroker.utils.normalize_hostnames(server_load)
+ sln_keys = server_load_n.keys()
+ esl_keys = expected_server_load.keys()
+ sln_keys.sort()
+ esl_keys.sort()
+ self.assertEqual(esl_keys, sln_keys)
+ self.assertEqual([], subdomains)
+
+ server_load = {'ts01.intern': 10, 'ts02.intern': 20, 'ts03.intern': 30,}
+ expected_server_load = {'ts01': 10, 'ts02': 20, 'ts03': 30,}
+ server_load_n, subdomains = x2gobroker.utils.normalize_hostnames(server_load)
+ sln_keys = server_load_n.keys()
+ esl_keys = expected_server_load.keys()
+ sln_keys.sort()
+ esl_keys.sort()
+ self.assertEqual(esl_keys, sln_keys)
+ self.assertEqual(['intern'], subdomains)
+
+ server_load = {'ts01.intern': 10, 'ts02.intern': 20, 'ts03.extern': 30,}
+ expected_server_load = {'ts01.intern': 10, 'ts02.intern': 20, 'ts03.extern': 30,}
+ server_load_n, subdomains = x2gobroker.utils.normalize_hostnames(server_load)
+ sln_keys = server_load_n.keys()
+ esl_keys = expected_server_load.keys()
+ sln_keys.sort()
+ esl_keys.sort()
+ subdomains.sort()
+ self.assertEqual(esl_keys, sln_keys)
+ self.assertEqual(['extern', 'intern'], subdomains)
+
+
+def test_suite():
+ from unittest import TestSuite, makeSuite
+ suite = TestSuite()
+ suite.addTest(makeSuite(TestX2GoBrokerUtils))
+ return suite
diff --git a/x2gobroker/tests/test_web_plain_inifile.py b/x2gobroker/tests/test_web_plain_inifile.py
index 114e73f..f87f81b 100644
--- a/x2gobroker/tests/test_web_plain_inifile.py
+++ b/x2gobroker/tests/test_web_plain_inifile.py
@@ -21,14 +21,14 @@ import unittest
import tempfile
from paste.fixture import TestApp
from nose.tools import *
-import web
+import tornado.wsgi
# Python X2GoBroker modules
import x2gobroker.defaults
import x2gobroker.web.plain
-urls = ( '/plain/(.*)', 'x2gobroker.web.plain.X2GoBrokerWeb',)
-app = web.application(urls, globals())
+urls = ( ('/plain/(.*)', x2gobroker.web.plain.X2GoBrokerWeb,) ,)
+application = tornado.wsgi.WSGIApplication(urls)
x2gobroker.defaults.X2GOBROKER_CONFIG_DEFAULTS.update({'base': {'enable': True, },})
diff --git a/x2gobroker/utils.py b/x2gobroker/utils.py
index 79f5ed3..0874cf5 100644
--- a/x2gobroker/utils.py
+++ b/x2gobroker/utils.py
@@ -23,6 +23,7 @@ import os
import sys
import types
import locale
+import netaddr
import distutils.version
def _checkConfigFileDefaults(data_structure):
@@ -103,3 +104,47 @@ def compare_versions(version_a, op, version_b):
return eval("ver_a %s ver_b" % op)
+
+def normalize_hostnames(servers):
+ """\
+ """
+
+ # test the data type of servers
+ arg_is_dict = False
+ servers_normalized = []
+ if type(servers) is types.DictType:
+ arg_is_dict = True
+ servers_normalized = {}
+ elif type(servers) is types.TupleType:
+ servers=list(servers)
+ elif type(servers) not in (types.ListType, types.TupleType):
+ raise ValueError('only lists, tuples and dictionaries are valid for x2gobroker.utils.normalize_hostnames()')
+
+ subdomains = []
+ for server in servers:
+
+ # do not deal with IPv4 or IPv6 addresses
+ if netaddr.valid_ipv4(server) or netaddr.valid_ipv6(server):
+ continue
+ else:
+ _server = server
+ if '.' not in _server:
+ _server += '.'
+ hostname, subdomain = _server.split('.', 1)
+ if arg_is_dict:
+ servers_normalized[hostname] = servers[server]
+ else:
+ servers_normalized.append(hostname)
+
+ # collect the list of subdomains used in all server names
+ if subdomain and subdomain not in subdomains:
+ subdomains.append(subdomain)
+ # stop processing if we have more than one subdomain
+ if len(subdomains) > 1:
+ break
+
+ # return the original servers dict/list/tuple
+ if len(subdomains) > 1:
+ servers_normalized = servers
+
+ return servers_normalized, subdomains
diff --git a/x2gobroker/web/extras.py b/x2gobroker/web/extras.py
index df48008..e1adf2a 100644
--- a/x2gobroker/web/extras.py
+++ b/x2gobroker/web/extras.py
@@ -40,7 +40,7 @@ class X2GoBrokerPubKeyService(tornado.web.RequestHandler):
def _gen_http_header(self):
for http_header_item in self.http_header_items.keys():
- self.add_header(http_header_item, self.http_header_items[http_header_item])
+ self.set_header(http_header_item, self.http_header_items[http_header_item])
def GET(self):
diff --git a/x2gobroker/web/html.py b/x2gobroker/web/html.py
index a604239..fd76eb8 100644
--- a/x2gobroker/web/html.py
+++ b/x2gobroker/web/html.py
@@ -62,7 +62,7 @@ $output
def _gen_http_header(self):
for http_header_item in self.http_header_items.keys():
- self.add_header(http_header_item, self.http_header_items[http_header_item])
+ self.set_header(http_header_item, self.http_header_items[http_header_item])
def get(self, backend):
self._gen_http_header()
diff --git a/x2gobroker/web/plain.py b/x2gobroker/web/plain.py
index f85bc98..c4deb42 100644
--- a/x2gobroker/web/plain.py
+++ b/x2gobroker/web/plain.py
@@ -40,7 +40,7 @@ class X2GoBrokerWeb(tornado.web.RequestHandler):
def _gen_http_header(self):
for http_header_item in self.http_header_items.keys():
- self.add_header(http_header_item, self.http_header_items[http_header_item])
+ self.set_header(http_header_item, self.http_header_items[http_header_item])
def get(self, backend):
if x2gobroker.defaults.X2GOBROKER_DEBUG:
hooks/post-receive
--
x2gobroker.git (HTTP(S) Session broker for X2Go)
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "x2gobroker.git" (HTTP(S) Session broker for X2Go).
