[X2go-Commits] x2gobroker.git - build-main (branch) updated: 0.0.0.7

X2Go dev team git-admin at x2go.org
Thu Mar 7 07:15:38 CET 2013


The branch, build-main has been updated
       via  15fba676bee4d4cebb9d44afe30259ca282ea7ef (commit)
       via  42032bd1bbcef0555bb15baf022704c09d94c342 (commit)
       via  26629de904cb2dc49f576d156efd6913ab7cccdd (commit)
       via  9ffb7417721a1ca2ab19e48cca2a545c92a56e0e (commit)
       via  02b425048b6beca03c0ae202d54005157dc71d5f (commit)
       via  38e1f3976082c99519843ab8f4cfa0730bdd818c (commit)
       via  64282631fe19aa8bea452a059edbff1d36b8d425 (commit)
       via  7f25682383416a9b0487d5cc0268dae9a8bdd0b4 (commit)
       via  8e73d26ce1bc5b1f8f1ce314877f90b37e6e8898 (commit)
       via  fe61669b7ab11de66223f62c52f380fc2b1e43bf (commit)
       via  e48dab769735322633d28a54c19f25bdeed27aa3 (commit)
       via  09dff9341cb3dc697bce90864fe3e9c76f059269 (commit)
       via  2573ffdec27908663f7ae786e6e9af4230dc69a8 (commit)
       via  9557b67570875897d1f45f61f174461c45b1a7c6 (commit)
       via  d4054a17444c93618020c4412e8b49361371f3cf (commit)
       via  140299b062adce94b65dc10ac19bb97497af87cc (commit)
       via  3f3e1d0961d98a8a4b993efc0ed06b0295f6bcac (commit)
       via  3f866e07a2c228fd3681a011f49794b73b51692a (commit)
      from  8676299cde99da2c19307431c2a81f46b16b31dc (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
-----------------------------------------------------------------------

Summary of changes:
 debian/changelog                           |   17 +++++
 debian/control                             |    2 +
 sbin/x2gobroker-pubkeyauthorizer           |    2 +-
 x2gobroker/__init__.py                     |   11 ++-
 x2gobroker/agent.py                        |    6 +-
 x2gobroker/brokers/base_broker.py          |  106 ++++++++++++++++++++++++----
 x2gobroker/tests/test_utils.py             |  102 ++++++++++++++++++++++++++
 x2gobroker/tests/test_web_plain_inifile.py |    6 +-
 x2gobroker/utils.py                        |   45 ++++++++++++
 x2gobroker/web/extras.py                   |    2 +-
 x2gobroker/web/html.py                     |    2 +-
 x2gobroker/web/plain.py                    |    2 +-
 12 files changed, 280 insertions(+), 23 deletions(-)
 create mode 100644 x2gobroker/tests/test_utils.py

The diff of changes is:
diff --git a/debian/changelog b/debian/changelog
index 872f3aa..7b5e8b0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,20 @@
+x2gobroker (0.0.0.7-0~x2go1) unstable; urgency=low
+
+  [ Mike Gabriel ]
+  * New upstream version (0.0.0.7):
+    - Add algorithm to ,,normalize'' hostnames used in session profiles
+      vs. those returned by the broker agent. (Fixes: #133).
+    - Ignore off-line X2Go servers in multi-node load-balanced setups.
+      (Fixes: #132).
+    - Return some sane output to x2goclient if the / all configured X2Go
+      server(s) is/are down.
+    - Tornado: Use RequestHandler.set_header() instead of
+      RequestHandler.add_header().
+  * /debian/control:
+    + Build-Depend on python-paste, python-nose (testsuite needs them).
+
+ -- Mike Gabriel <mike.gabriel at das-netzwerkteam.de>  Thu, 07 Mar 2013 07:14:31 +0100
+
 x2gobroker (0.0.0.6-0~x2go1) unstable; urgency=low
 
   [ Mike Gabriel ]
diff --git a/debian/control b/debian/control
index ece5439..79dc5cb 100644
--- a/debian/control
+++ b/debian/control
@@ -11,6 +11,8 @@ Build-Depends:
  dpkg-dev (>= 1.16.1~),
  python (>= 2.6.5-0~),
  python-setuptools,
+ python-nose,
+ python-paste,
 Standards-Version: 3.9.3
 XS-Python-Version: >= 2.4
 
diff --git a/sbin/x2gobroker-pubkeyauthorizer b/sbin/x2gobroker-pubkeyauthorizer
index c1e9f41..39440bb 100755
--- a/sbin/x2gobroker-pubkeyauthorizer
+++ b/sbin/x2gobroker-pubkeyauthorizer
@@ -36,7 +36,7 @@ import logging.config
 from pwd import getpwnam
 from grp import getgrnam
 
-__VERSION__ = '0.0.0.6'
+__VERSION__ = '0.0.0.7'
 __AUTHOR__ = 'Mike Gabriel (X2Go Project) <mike.gabriel at das-netzwerkteam.de>'
 
 PROG_NAME = os.path.basename(sys.argv[0])
diff --git a/x2gobroker/__init__.py b/x2gobroker/__init__.py
index 2c644a2..f63bf9c 100644
--- a/x2gobroker/__init__.py
+++ b/x2gobroker/__init__.py
@@ -18,5 +18,14 @@
 # Free Software Foundation, Inc.,
 # 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
 
-__VERSION__ = '0.0.0.6'
+__VERSION__ = '0.0.0.7'
 __AUTHOR__ = 'Mike Gabriel (X2Go Project) <mike.gabriel at das-netzwerkteam.de>'
+
+from loggers import logger_error
+
+class X2GoBrokerBaseException(BaseException):
+    def __init__(self, *args, **kwargs):
+        BaseException.__init__(self, *args, **kwargs)
+        logger_error.error('An exceptional problem occurred: {exception}("{msg}")'.format(exception=type(self).__name__, msg=str(self)))
+
+class X2GoBrokerAgentException(X2GoBrokerBaseException): pass
diff --git a/x2gobroker/agent.py b/x2gobroker/agent.py
index 9b63f00..001a800 100644
--- a/x2gobroker/agent.py
+++ b/x2gobroker/agent.py
@@ -24,6 +24,7 @@ import paramiko
 import cStringIO
 import time
 import threading
+import socket
 
 import x2gobroker._paramiko
 x2gobroker._paramiko.monkey_patch_paramiko()
@@ -82,6 +83,7 @@ def call_local_broker_agent(username, mode, cmdline_args=[]):
     if result[0].startswith('OK'):
         return [ r for r in result[1:] if r ]
 
+    raise x2gobroker.X2GoBrokerAgentException('Query to local X2Go Broker Agent failed with no response')
 
 def call_remote_broker_agent(username, mode, cmdline_args=[], remote_agent=None):
     """\
@@ -132,8 +134,8 @@ def call_remote_broker_agent(username, mode, cmdline_args=[], remote_agent=None)
         client.close()
         if result and result[0].startswith('OK'):
             return [ r for r in result[1:] if r ]
-    except paramiko.SSHException:
-        logger_error.error('could not connect to remote X2Go Broker Agent (user: {user}, hostname: {hostname}, port: {port}'.format(user=remote_username, hostname=remote_hostname, port=remote_port))
+    except (paramiko.SSHException, paramiko.AuthenticationException, paramiko.BadHostKeyException, socket.error):
+        raise x2gobroker.X2GoBrokerAgentException('Query to remote X2Go Broker Agent (user: {user}, hostname: {hostname}, port: {port}) failed'.format(user=remote_username, hostname=remote_hostname, port=remote_port))
 
 
 def list_sessions(username, query_mode='LOCAL', remote_agent=None):
diff --git a/x2gobroker/brokers/base_broker.py b/x2gobroker/brokers/base_broker.py
index 9001039..89aadac 100644
--- a/x2gobroker/brokers/base_broker.py
+++ b/x2gobroker/brokers/base_broker.py
@@ -787,6 +787,31 @@ class X2GoBroker(object):
 
         return list_of_profiles
 
+    def random_remote_agent(self, profile_id, exclude_agents=[]):
+        """\
+        Randomly choose a remote agent for agent query.
+
+        @param profile_id: choose remote agent for this profile ID
+        @type profile_id: C{unicode}
+        @param exclude_agents: a list of remote agent dict objects to be exclude from the random choice
+        @type exclude_agents: C{list}
+
+        """
+        profile = self.get_profile(profile_id)
+        server_list = profile[u'host']
+
+        for agent in exclude_agents:
+            if agent['hostname'] in server_list:
+                server_list.remove(agent['hostname'])
+
+        remote_agent = None
+        if server_list:
+            random.shuffle(server_list)
+            remote_agent_server = server_list[0]
+            remote_agent_port = profile[u'sshport']
+            remote_agent = {u'hostname': remote_agent_server, u'port': remote_agent_port, }
+        return remote_agent
+
     def select_session(self, profile_id, username=None):
         """\
         Start/resume a session by selecting a profile name offered by the X2Go client.
@@ -804,30 +829,85 @@ class X2GoBroker(object):
 
         # if we have more than one server, pick one server randomly for X2Go Broker Agent queries
         server_list = profile[u'host']
+        random.shuffle(server_list)
         agent_query_mode = self.get_agent_query_mode()
 
         remote_agent = None
         if agent_query_mode.upper() == u'SSH':
-            random.shuffle(server_list)
-            remote_agent_server = server_list[0]
-            remote_agent_port = profile[u'sshport']
-            remote_agent = {u'hostname': remote_agent_server, u'port': remote_agent_port, }
+            remote_agent = self.random_remote_agent(profile_id)
 
+        # detect best X2Go server for this user if load balancing is configured
         if len(server_list) >= 2 and username:
 
-            busy_servers = x2gobroker.agent.find_busy_servers(username=username, query_mode=agent_query_mode, remote_agent = remote_agent)
+            # query remote agent for session info, if one of the server's is down, we will try the next one...
+            busy_servers = None
+            exclude_agents = []
+            while busy_servers is None and remote_agent:
+
+                try:
+                    busy_servers = x2gobroker.agent.find_busy_servers(username=username, query_mode=agent_query_mode, remote_agent=remote_agent)
+                except x2gobroker.X2GoBrokerAgentException:
+                    logger_broker.warning('base_broker.X2GoBroker.select_session(): failed to query broker agent (quey-mode: {query_mode}, remote_agent: {remote_agent})'.format(query_mode=agent_query_mode, remote_agent=remote_agent))
+
+                    if agent_query_mode.upper() == u'SSH':
+                        # mark this agent as bad
+                        exclude_agents.append(remote_agent)
+                        # also remove this agent from the list of available servers as the machine is probably down
+                        server_list.remove(remote_agent['hostname'])
+
+                        remote_agent = self.random_remote_agent(profile_id, exclude_agents=exclude_agents)
+                    else:
+                        remote_agent = None
+
+            if busy_servers is not None:
+
+                # if we do not get until here, then all broker agent calls have failed!!!!
+
+                # when detecting the server load we have to support handling of differing subdomains (config
+                # file vs. server load returned by x2gobroker agent). Best approach: all members of a multi-node
+                # server farm either (a) do not have a subdomain in their hostname or (b) the subdomain is
+                # is identical.
+
+                # Example:
+                #
+                #    ts01, ts02 - hostnames as returned by agent
+                #    ts01.intern, ts02.intern - hostnames configured in session profile option ,,host''
+                #    -> this will result in the subdomain .intern being stripped off from the hostnames before detecting
+                #       the best server for this user
 
-            for server in server_list:
-                if server not in busy_servers.keys():
-                    busy_servers[server] = 0
+                ### NORMALIZE (=reduce to hostname only) X2Go server names (as found in config) if possible
+                server_list_normalized, subdomains_config = x2gobroker.utils.normalize_hostnames(server_list)
 
-            busy_server_list = [ (load, server) for server, load in busy_servers.items() ]
-            busy_server_list.sort()
+                ### NORMALIZE X2Go server names (as returned by x2gobroker agent)--only if the hostnames in the config share the same subdomain
+                if len(subdomains_config) == 1:
 
-            best_server = busy_server_list[0][1]
+                    busy_servers_normalized, subdomains_agent = x2gobroker.utils.normalize_hostnames(busy_servers)
+                    if len(subdomains_agent) <= 1:
+
+                        # all X2Go servers in the multi-node server farm are in the same DNS subdomain
+                        # we can operate on hostname-only hostnames
+                        server_list = server_list_normalized
+                        busy_servers = busy_servers_normalized
+
+                for server in server_list:
+                    if server not in busy_servers.keys():
+                        busy_servers[server] = 0
+
+                busy_server_list = [ (load, server) for server, load in busy_servers.items() ]
+                busy_server_list.sort()
+
+                logger_broker.debug('base_broker.X2GoBroker.select_session(): load balancer analysis: {server_load}'.format(server_load=unicode(busy_server_list)))
+
+                best_server = busy_server_list[0][1]
+
+            else:
+                logger_broker.warning('base_broker.X2GoBroker.select_session(): all expected broker agents failed to respond, this does not look good. We tried these agent hosts: {agent_hosts}'.format(agent_hosts=unicode(server_list)))
+                if server_list: best_server = server_list[0]
+                else: return { 'server': 'no-server-available', 'port': profile[u'sshport'], }
 
         else:
-            best_server = server_list[0]
+            if server_list: best_server = server_list[0]
+            else: return { 'server': 'no-server-available', 'port': profile[u'sshport'], }
 
         selected_session = {
             'server': best_server,
@@ -877,7 +957,7 @@ class X2GoBroker(object):
                                                    query_mode=agent_query_mode,
                                                    remote_agent=remote_agent,
                                                    delay_deletion=20,
-            ),
+            )
 
         return selected_session
 
diff --git a/x2gobroker/tests/test_utils.py b/x2gobroker/tests/test_utils.py
new file mode 100644
index 0000000..871dca7
--- /dev/null
+++ b/x2gobroker/tests/test_utils.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012 by Mike Gabriel <mike.gabriel at das-netzwerkteam.de>
+#
+# X2Go Session Broker is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# X2Go Session Broker is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program; if not, write to the
+# Free Software Foundation, Inc.,
+# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+import unittest
+import tempfile
+
+# Python X2GoBroker modules
+import x2gobroker.utils
+
+class TestX2GoBrokerUtils(unittest.TestCase):
+
+    ### TEST FUNCTION: normalize_hostnames() with server lists (ListType)
+
+    def test_normalize_hostnames_listtype(self):
+
+        server_list = ['ts01', 'ts02', 'ts03',]
+        expected_server_list = ['ts01', 'ts02', 'ts03',]
+        server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_list)
+        server_list_n.sort()
+        self.assertEqual(expected_server_list, server_list_n)
+        self.assertEqual([], subdomains)
+
+        server_list = ['ts01.intern', 'ts02.intern', 'ts03.intern',]
+        expected_server_list = ['ts01', 'ts02', 'ts03',]
+        server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_list)
+        server_list_n.sort()
+        self.assertEqual(expected_server_list, server_list_n)
+        self.assertEqual(['intern'], subdomains)
+
+        server_list = ['ts01.intern', 'ts02.intern', 'ts03.extern',]
+        expected_server_list = ['ts01.intern', 'ts02.intern', 'ts03.extern',]
+        server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_list)
+        server_list_n.sort()
+        subdomains.sort()
+        self.assertEqual(expected_server_list, server_list_n)
+        self.assertEqual(['extern', 'intern'], subdomains)
+
+        server_tuple = ('ts01.intern', 'ts02.intern', 'ts03.extern',)
+        expected_server_list = ['ts01.intern', 'ts02.intern', 'ts03.extern',]
+        server_list_n, subdomains = x2gobroker.utils.normalize_hostnames(server_tuple)
+        server_list_n.sort()
+        subdomains.sort()
+        self.assertEqual(expected_server_list, server_list_n)
+        self.assertEqual(['extern', 'intern'], subdomains)
+
+    ### TEST FUNCTION: normalize_hostnames() with server load (DictType)
+
+    def test_normalize_hostnames_dicttype(self):
+
+        server_load = {'ts01': 10, 'ts02': 20, 'ts03': 30,}
+        expected_server_load = {'ts01': 10, 'ts02': 20, 'ts03': 30,}
+        server_load_n, subdomains = x2gobroker.utils.normalize_hostnames(server_load)
+        sln_keys = server_load_n.keys()
+        esl_keys = expected_server_load.keys()
+        sln_keys.sort()
+        esl_keys.sort()
+        self.assertEqual(esl_keys, sln_keys)
+        self.assertEqual([], subdomains)
+
+        server_load = {'ts01.intern': 10, 'ts02.intern': 20, 'ts03.intern': 30,}
+        expected_server_load = {'ts01': 10, 'ts02': 20, 'ts03': 30,}
+        server_load_n, subdomains = x2gobroker.utils.normalize_hostnames(server_load)
+        sln_keys = server_load_n.keys()
+        esl_keys = expected_server_load.keys()
+        sln_keys.sort()
+        esl_keys.sort()
+        self.assertEqual(esl_keys, sln_keys)
+        self.assertEqual(['intern'], subdomains)
+
+        server_load = {'ts01.intern': 10, 'ts02.intern': 20, 'ts03.extern': 30,}
+        expected_server_load = {'ts01.intern': 10, 'ts02.intern': 20, 'ts03.extern': 30,}
+        server_load_n, subdomains = x2gobroker.utils.normalize_hostnames(server_load)
+        sln_keys = server_load_n.keys()
+        esl_keys = expected_server_load.keys()
+        sln_keys.sort()
+        esl_keys.sort()
+        subdomains.sort()
+        self.assertEqual(esl_keys, sln_keys)
+        self.assertEqual(['extern', 'intern'], subdomains)
+
+
+def test_suite():
+    from unittest import TestSuite, makeSuite
+    suite = TestSuite()
+    suite.addTest(makeSuite(TestX2GoBrokerUtils))
+    return suite
diff --git a/x2gobroker/tests/test_web_plain_inifile.py b/x2gobroker/tests/test_web_plain_inifile.py
index 114e73f..f87f81b 100644
--- a/x2gobroker/tests/test_web_plain_inifile.py
+++ b/x2gobroker/tests/test_web_plain_inifile.py
@@ -21,14 +21,14 @@ import unittest
 import tempfile
 from paste.fixture import TestApp
 from nose.tools import *
-import web
+import tornado.wsgi
 
 # Python X2GoBroker modules
 import x2gobroker.defaults
 import x2gobroker.web.plain
 
-urls = ( '/plain/(.*)', 'x2gobroker.web.plain.X2GoBrokerWeb',)
-app = web.application(urls, globals())
+urls = ( ('/plain/(.*)', x2gobroker.web.plain.X2GoBrokerWeb,) ,)
+application = tornado.wsgi.WSGIApplication(urls)
 
 x2gobroker.defaults.X2GOBROKER_CONFIG_DEFAULTS.update({'base': {'enable': True, },})
 
diff --git a/x2gobroker/utils.py b/x2gobroker/utils.py
index 79f5ed3..0874cf5 100644
--- a/x2gobroker/utils.py
+++ b/x2gobroker/utils.py
@@ -23,6 +23,7 @@ import os
 import sys
 import types
 import locale
+import netaddr
 import distutils.version
 
 def _checkConfigFileDefaults(data_structure):
@@ -103,3 +104,47 @@ def compare_versions(version_a, op, version_b):
 
     return eval("ver_a %s ver_b" % op)
 
+
+def normalize_hostnames(servers):
+    """\
+    """
+
+    # test the data type of servers
+    arg_is_dict = False
+    servers_normalized = []
+    if type(servers) is types.DictType:
+        arg_is_dict = True
+        servers_normalized = {}
+    elif type(servers) is types.TupleType:
+        servers=list(servers)
+    elif type(servers) not in (types.ListType, types.TupleType):
+        raise ValueError('only lists, tuples and dictionaries are valid for x2gobroker.utils.normalize_hostnames()')
+
+    subdomains = []
+    for server in servers:
+
+        # do not deal with IPv4 or IPv6 addresses
+        if netaddr.valid_ipv4(server) or netaddr.valid_ipv6(server):
+            continue
+        else:
+            _server = server
+            if '.' not in _server:
+                _server += '.'
+            hostname, subdomain = _server.split('.', 1)
+            if arg_is_dict:
+                servers_normalized[hostname] = servers[server]
+            else:
+                servers_normalized.append(hostname)
+
+            # collect the list of subdomains used in all server names
+            if subdomain and subdomain not in subdomains:
+                subdomains.append(subdomain)
+            # stop processing if we have more than one subdomain
+            if len(subdomains) > 1:
+                break
+
+    # return the original servers dict/list/tuple
+    if len(subdomains) > 1:
+        servers_normalized = servers
+
+    return servers_normalized, subdomains
diff --git a/x2gobroker/web/extras.py b/x2gobroker/web/extras.py
index df48008..e1adf2a 100644
--- a/x2gobroker/web/extras.py
+++ b/x2gobroker/web/extras.py
@@ -40,7 +40,7 @@ class X2GoBrokerPubKeyService(tornado.web.RequestHandler):
     def _gen_http_header(self):
 
         for http_header_item in self.http_header_items.keys():
-            self.add_header(http_header_item, self.http_header_items[http_header_item])
+            self.set_header(http_header_item, self.http_header_items[http_header_item])
 
     def GET(self):
 
diff --git a/x2gobroker/web/html.py b/x2gobroker/web/html.py
index a604239..fd76eb8 100644
--- a/x2gobroker/web/html.py
+++ b/x2gobroker/web/html.py
@@ -62,7 +62,7 @@ $output
     def _gen_http_header(self):
 
         for http_header_item in self.http_header_items.keys():
-            self.add_header(http_header_item, self.http_header_items[http_header_item])
+            self.set_header(http_header_item, self.http_header_items[http_header_item])
 
     def get(self, backend):
         self._gen_http_header()
diff --git a/x2gobroker/web/plain.py b/x2gobroker/web/plain.py
index f85bc98..c4deb42 100644
--- a/x2gobroker/web/plain.py
+++ b/x2gobroker/web/plain.py
@@ -40,7 +40,7 @@ class X2GoBrokerWeb(tornado.web.RequestHandler):
     def _gen_http_header(self):
 
         for http_header_item in self.http_header_items.keys():
-            self.add_header(http_header_item, self.http_header_items[http_header_item])
+            self.set_header(http_header_item, self.http_header_items[http_header_item])
 
     def get(self, backend):
         if x2gobroker.defaults.X2GOBROKER_DEBUG:


hooks/post-receive
-- 
x2gobroker.git (HTTP(S) Session broker for X2Go)

This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "x2gobroker.git" (HTTP(S) Session broker for X2Go).




More information about the x2go-commits mailing list