diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 127d4f56c9058584182198996238ca9619ebdc83..88c99d1bd55843cd07e6439227f74aa2691c598f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ stages: - prebuild - test - publish +- deploy prebuild-docker-image: image: docker:stable @@ -129,3 +130,47 @@ publish_pypi: - pip install wheel twine --upgrade - python setup.py sdist bdist_wheel - twine upload dist/* -u $PYPI_USR -p $PYPI_PWD + +rdiffweb_deploy_staging: + stage: deploy + image: ikus060/ansible + environment: + name: staging + url: https://sestican.patrikdufresne.com + variables: + ANSIBLE_HOST_KEY_CHECKING: "False" + before_script: + - eval $(ssh-agent -s) + - echo "$SESTICAN_ROOT_PRIVATEKEY" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + script: + - python ./setup.py --version + - export RDIFFWEB_VERSION=$(python ./setup.py --version) + - git clone "http://${GITLAB_USR}:${GITLAB_PWD}@git.patrikdufresne.com/pdsl/ansible-config.git" + - cd ansible-config + - ansible-playbook minarca.yml -i pdsl --extra-vars "ansible_user=root rdiffweb_version=$RDIFFWEB_VERSION rdiffweb_default_repositories=true" --limit sestican + +rdiffweb_deploy_prod: + stage: deploy + when: manual + only: + - tags + image: ikus060/ansible + environment: + name: prod + url: https://www.minarca.net + variables: + ANSIBLE_HOST_KEY_CHECKING: "False" + before_script: + - eval $(ssh-agent -s) + - echo "$RANCULOS_ROOT_PRIVATEKEY" | tr -d '\r' | ssh-add - > /dev/null + - echo "$MERCOR_ROOT_PRIVATEKEY" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + script: + - python ./setup.py --version + - export RDIFFWEB_VERSION=$(python ./setup.py --version) + - git clone "http://${GITLAB_USR}:${GITLAB_PWD}@git.patrikdufresne.com/pdsl/ansible-config.git" + - cd ansible-config + - ansible-playbook minarca.yml -i pdsl --extra-vars "ansible_user=root rdiffweb_version=$RDIFFWEB_VERSION" --limit ranculos,mercor diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index acd1dd834c105ae0f6e2a8e3409cd2c7cb829244..0000000000000000000000000000000000000000 --- a/.travis.yml +++ /dev/null @@ -1,41 +0,0 @@ -sudo: false -language: python -python: - - "2.7" - - "3.4" - -env: - global: - - TMPDIR=/var/tmp - matrix: - - CHERRYPY=3.2 - - CHERRYPY=3.5 - - CHERRYPY=4 - - CHERRYPY=5 - - CHERRYPY=6 - - CHERRYPY=7 - - CHERRYPY=8 - - CHERRYPY=9 - - CHERRYPY=10 - - CHERRYPY=11 - - CHERRYPY=12 - - CHERRYPY=13 - - CHERRYPY=14 - - CHERRYPY=15 - - CHERRYPY=16 - -# Make sure to exclude python3.4 + CherryPy 3.2.2 -# This configuration is not supported. -matrix: - exclude: - - python: "3.4" - env: CHERRYPY=3.2 - -before_install: - # Install requirements - - sudo apt-get install python-pysqlite2 rdiff-backup - - sudo pip install tox-travis -# Need to compile the catalogs before running the test -script: - - python setup.py compile_all_catalogs - - tox \ No newline at end of file diff --git a/rdiffweb/filter_authentication.py b/rdiffweb/filter_authentication.py index 9d2e2c41d59c1b370bb9884c870409fced7e43c8..fd94e962f164c37d66711ecd23bb46a3865899d1 100644 --- a/rdiffweb/filter_authentication.py +++ b/rdiffweb/filter_authentication.py @@ -54,7 +54,7 @@ class BaseAuth(HandlerTool): def check_username_and_password(self, username, password): """Validate user credentials.""" - logger.info("check credentials for [%s]", username) + logger.debug("check credentials for [%s]", username) try: userobj = cherrypy.request.app.userdb.login(username, password) # @UndefinedVariable except: @@ -183,7 +183,7 @@ class AuthFormTool(BaseAuth): if path.startswith(native_str('/login')): if request.method != 'POST': response.headers['Allow'] = "POST" - logger.warn('do_login requires POST, redirect to /') + logger.debug('/login requires POST, redirect to /') # Redirect to / instead of showing error. raise cherrypy.HTTPRedirect(b'/') logger.debug('routing %(path)r to do_login', locals()) @@ -194,7 +194,6 @@ class AuthFormTool(BaseAuth): return self.do_logout(**request.params) # No special path, validate session. - logger.debug('no special path, running do_check') return self.do_check() @@ -224,7 +223,7 @@ class BasicAuth(BaseAuth): if scheme.lower() == 'basic': # Validate user credential. login, password = base64_decode(params).split(':', 1) - logger.info('routing %(path)r to do_login', locals()) + logger.debug('routing %(path)r to do_login', locals()) try: return self.do_login(login, password) except RdiffError as e: @@ -234,7 +233,7 @@ class BasicAuth(BaseAuth): except (ValueError, binascii.Error): raise cherrypy.HTTPError(400, 'Bad Request') - logger.info('no authorization header, running is_login') + logger.debug('no authorization header, running is_login') if not self.is_login(): # Inform the user-agent this path is protected. cherrypy.serving.response.headers['www-authenticate'] = ( diff --git a/rdiffweb/main.py b/rdiffweb/main.py index 9a3fd2cd45031c38043b7860b08299a51ae07881..4cc870a846f4169edfefff9bb1ebda2dcc0f1be4 100755 --- a/rdiffweb/main.py +++ b/rdiffweb/main.py @@ -16,12 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function from __future__ import unicode_literals import cherrypy from future.builtins import str import getopt import logging +import os import sys import tempfile import threading @@ -30,7 +32,6 @@ import traceback from rdiffweb import rdw_app, rdw_config from rdiffweb.rdw_profiler import ProfilingApplication - # Define logger for this module logger = logging.getLogger(__name__) @@ -117,6 +118,7 @@ def setup_logging(log_file, log_access_file, level): # Configure default log file. if log_file: assert isinstance(log_file, str) + print("continue logging to %s" % log_file) logging.basicConfig(filename=log_file, level=level, format=logformat) else: logging.basicConfig(level=level, format=logformat) @@ -124,6 +126,7 @@ def setup_logging(log_file, log_access_file, level): # Configure access log file. if log_access_file: assert isinstance(log_access_file, str) + print("continue logging access to %s" % log_access_file) logging.root.handlers[0].addFilter(NotFilter("cherrypy.access")) logging.root.handlers[0].addFilter(ContextFilter()) @@ -167,15 +170,18 @@ def start(): # Open config file before opening the apps. configfile = args.get('config', '/etc/rdiffweb/rdw.conf') - tmp_cfg = rdw_config.Configuration(configfile) - log_file = args.get('log_file', None) or tmp_cfg.get_config('LogFile', False) - log_access_file = args.get('log_access_file', None) or tmp_cfg.get_config('LogAccessFile', False) + if not os.path.isfile(configfile): + print("configuration file %s doesn't exists" % configfile, file=sys.stderr) + exit(1) + cfg = rdw_config.Configuration(configfile) + log_file = args.get('log_file', None) or cfg.get_config('LogFile', False) + log_access_file = args.get('log_access_file', None) or cfg.get_config('LogAccessFile', None) if args.get('debug', False): environment = 'development' log_level = "DEBUG" else: - environment = tmp_cfg.get_config('Environment', 'production') - log_level = tmp_cfg.get_config('LogLevel', 'INFO') + environment = cfg.get_config('Environment', 'production') + log_level = cfg.get_config('LogLevel', 'INFO') # Configure logging setup_logging( @@ -187,10 +193,10 @@ def start(): logger.info("START") # Create App. - app = rdw_app.RdiffwebApp(configfile) + app = rdw_app.RdiffwebApp(cfg) # Get configuration - serverHost = app.cfg.get_config("ServerHost", default="0.0.0.0") + serverHost = app.cfg.get_config("ServerHost", default=b"0.0.0.0") serverPort = app.cfg.get_config_int("ServerPort", default="8080") # Get SSL configuration (if any) sslCertificate = app.cfg.get_config("SslCertificate") diff --git a/rdiffweb/plugins/ldap_auth/__init__.py b/rdiffweb/plugins/ldap_auth/__init__.py index 2d4a0ae5d23aefcce4d4cd7556b6f9a072a19980..6b9bbaef2e7415cd3d4ffe648ddca2149554f381 100644 --- a/rdiffweb/plugins/ldap_auth/__init__.py +++ b/rdiffweb/plugins/ldap_auth/__init__.py @@ -240,7 +240,7 @@ class LdapPasswordStore(IPasswordStore): def fetch_user_email(l, r): # @UnusedVariable if len(r) != 1: - logger.warning("user [%s] not found", username) + logger.warning("user [%s] not found in LDAP", username) return "" return self._attr(r, attr) diff --git a/rdiffweb/rdw_app.py b/rdiffweb/rdw_app.py index 6bf2a87b64ddbb9e8991beddb831da723e8188b4..3264cafb48df358cd5ee787a7596910001c2346f 100644 --- a/rdiffweb/rdw_app.py +++ b/rdiffweb/rdw_app.py @@ -31,11 +31,11 @@ import sys from rdiffweb import filter_authentication # @UnusedImport from rdiffweb import i18n # @UnusedImport -from rdiffweb import rdw_config, page_main +from rdiffweb import page_main from rdiffweb import rdw_plugin from rdiffweb import rdw_templating from rdiffweb.api import ApiPage -from rdiffweb.dispatch import static, empty +from rdiffweb.dispatch import static, empty # @UnusedImport from rdiffweb.page_admin import AdminPage from rdiffweb.page_browse import BrowsePage from rdiffweb.page_history import HistoryPage @@ -45,14 +45,12 @@ from rdiffweb.page_restore import RestorePage from rdiffweb.page_settings import SettingsPage from rdiffweb.page_status import StatusPage from rdiffweb.user import UserManager -from rdiffweb.page_main import MainPage +from rdiffweb.page_main import MainPage # @UnusedImport from rdiffweb.librdiff import DoesNotExistError, AccessDeniedError - # Define the logger logger = logging.getLogger(__name__) - PY3 = sys.version_info[0] == 3 @@ -91,10 +89,16 @@ class Root(LocationsPage): class RdiffwebApp(Application): """This class represent the application context.""" - def __init__(self, configfile=None): + def __init__(self, cfg): # Initialise the configuration - self.load_config(configfile) + assert cfg + self.cfg = cfg + + # Define TEMP env + tempdir = self.cfg.get_config("TempDir", default="") + if tempdir: + os.environ["TMPDIR"] = tempdir # Initialise the template engine. self.templates = rdw_templating.TemplateManager() @@ -195,15 +199,6 @@ class RdiffwebApp(Application): self._version = "DEV" return self._version - def load_config(self, configfile=None): - """Called during app creating to load the configuration from file.""" - self.cfg = rdw_config.Configuration(configfile) - - # Define TEMP env - tempdir = self.cfg.get_config("TempDir", default="") - if tempdir: - os.environ["TMPDIR"] = tempdir - def _localedirs(self): """ Return a list of locales directory where to search for mo files. This diff --git a/rdiffweb/test.py b/rdiffweb/test.py index 824148ffa8ff10ae2a83c26e395271aaeec79e03..a6bb7e7506be2cfad25f91c6f9f2000ab741c059 100644 --- a/rdiffweb/test.py +++ b/rdiffweb/test.py @@ -37,9 +37,9 @@ import tarfile import tempfile import unittest +from rdiffweb import rdw_config from rdiffweb.rdw_app import RdiffwebApp - try: from urllib.parse import urlencode # @UnresolvedImport @UnusedImport except: @@ -47,14 +47,33 @@ except: class MockRdiffwebApp(RdiffwebApp): + def __init__(self, enabled_plugins=['SQLite'], default_config={}): assert enabled_plugins is None or isinstance(enabled_plugins, list) self.enabled_plugins = enabled_plugins assert default_config is None or isinstance(default_config, dict) self.default_config = default_config + # Define config + cfg = rdw_config.Configuration() + for plugin_name in self.enabled_plugins: + cfg.set_config('%sEnabled' % plugin_name, 'True') + + # database in memory + if 'SQLite' in self.enabled_plugins: + self.database_dir = tempfile.mkdtemp(prefix='rdiffweb_tests_db_') + cfg.set_config('SQLiteDBFile', os.path.join(self.database_dir, 'rdiffweb.tmp.db')) + + if 'Ldap' in self.enabled_plugins: + cfg.set_config('LdapUri', '__default__') + cfg.set_config('LdapBaseDn', 'dc=nodomain') + + # Set config + for key, val in list(self.default_config.items()): + cfg.set_config(key, val) + # Call parent constructor - RdiffwebApp.__init__(self) + RdiffwebApp.__init__(self, cfg) def clear_db(self): if hasattr(self, 'database_dir'): @@ -66,26 +85,6 @@ class MockRdiffwebApp(RdiffwebApp): shutil.rmtree(native_str(self.testcases)) delattr(self, 'testcases') - def load_config(self, configfile=None): - RdiffwebApp.load_config(self, None) - - # Enabled given plugins - for plugin_name in self.enabled_plugins: - self.cfg.set_config('%sEnabled' % plugin_name, 'True') - - # database in memory - if 'SQLite' in self.enabled_plugins: - self.database_dir = tempfile.mkdtemp(prefix='rdiffweb_tests_db_') - self.cfg.set_config('SQLiteDBFile', os.path.join(self.database_dir, 'rdiffweb.tmp.db')) - - if 'Ldap' in self.enabled_plugins: - self.cfg.set_config('LdapUri', '__default__') - self.cfg.set_config('LdapBaseDn', 'dc=nodomain') - - # Set config - for key, val in list(self.default_config.items()): - self.cfg.set_config(key, val) - def reset(self, username=None, password=None): """ Reset the application. Delete all data from database. diff --git a/rdiffweb/user.py b/rdiffweb/user.py index 238a2d8147014cdbd0d55237963895bd32234844..e6410c29615c245225f5af0ab2acefe415af5fd2 100644 --- a/rdiffweb/user.py +++ b/rdiffweb/user.py @@ -301,6 +301,8 @@ class UserManager(Component): assert password is None or isinstance(user, str) # Validate the credentials logger.debug("validating user [%s] credentials", user) + if not self._password_stores: + logger.warn("not password store available to validate user credentials") real_user = False for store in self._password_stores: real_user = store.are_valid_credentials(user, password) diff --git a/tox.ini b/tox.ini index e9a9d71726be5dc209597a972504ba251b929ad3..cc645887237c8ab9440e2fc7002bc0c1588b532b 100644 --- a/tox.ini +++ b/tox.ini @@ -39,21 +39,3 @@ deps= setenv = COVERAGE_FILE=.coverage.{envname} commands=python setup.py nosetests --xunit-file=nosetests-{envname}.xml --xunit-testsuite-name={envname} - -[travis:env] -CHERRYPY = - 3.2: cherrypy32 - 3.5: cherrypy35 - 4: cherrypy4 - 5: cherrypy5 - 6: cherrypy6 - 7: cherrypy7 - 8: cherrypy8 - 9: cherrypy9 - 10: cherrypy10 - 11: cherrypy11 - 12: cherrypy12 - 13: cherrypy13 - 14: cherrypy14 - 14: cherrypy15 - 14: cherrypy16 \ No newline at end of file