diff --git a/cmapi/cmapi_server/constants.py b/cmapi/cmapi_server/constants.py index 9fba94305d..464b61d99a 100644 --- a/cmapi/cmapi_server/constants.py +++ b/cmapi/cmapi_server/constants.py @@ -20,9 +20,11 @@ MCS_EM_PATH = os.path.join(MCS_DATA_PATH, EM_PATH_SUFFIX) MCS_BRM_CURRENT_PATH = os.path.join(MCS_EM_PATH, 'BRM_saves_current') S3_BRM_CURRENT_PATH = os.path.join(EM_PATH_SUFFIX, 'BRM_saves_current') + # keys file for CEJ password encryption\decryption # (CrossEngineSupport section in Columnstore.xml) -MCS_SECRETS_FILE_PATH = os.path.join(MCS_DATA_PATH, '.secrets') +MCS_SECRETS_FILENAME = '.secrets' +MCS_SECRETS_FILE_PATH = os.path.join(MCS_DATA_PATH, MCS_SECRETS_FILENAME) # CMAPI SERVER CMAPI_CONFIG_FILENAME = 'cmapi_server.conf' diff --git a/cmapi/cmapi_server/controllers/api_clients.py b/cmapi/cmapi_server/controllers/api_clients.py index c03d0b2635..dd449258ab 100644 --- a/cmapi/cmapi_server/controllers/api_clients.py +++ b/cmapi/cmapi_server/controllers/api_clients.py @@ -1,7 +1,13 @@ import requests from typing import Any, Dict, Optional, Union + +import pyotp + from cmapi_server.controllers.dispatcher import _version -from cmapi_server.constants import CURRENT_NODE_CMAPI_URL +from cmapi_server.constants import ( + CMAPI_CONF_PATH, CURRENT_NODE_CMAPI_URL, SECRET_KEY, +) +from cmapi_server.helpers import get_config_parser, get_current_key class ClusterControllerClient: @@ -11,6 +17,10 @@ def __init__( ): """Initialize the ClusterControllerClient with the base URL. + WARNING: This class only handles the API requests, it does not + handle the transaction management. So it should be started + at level above using TransactionManager (decorator or context manager). + :param base_url: The base URL for the API endpoints, defaults to CURRENT_NODE_CMAPI_URL :type base_url: str, optional @@ -59,14 +69,14 @@ def add_node( return self._request('PUT', 'node', {**node_info, **extra}) def remove_node( - self, node_id: str, extra: Dict[str, Any] = dict() + self, node: str, extra: Dict[str, Any] = dict() ) -> Union[Dict[str, Any], Dict[str, str]]: """Remove a node from the cluster. - :param node_id: The ID of the node to remove. + :param node: node IP, name or FQDN. :return: The response from the API. """ - return self._request('DELETE', 'node', {'node_id': node_id}) + return self._request('DELETE', 'node', {'node': node, **extra}) def get_status(self) -> Union[Dict[str, Any], Dict[str, str]]: """Get the status of the cluster. @@ -83,7 +93,12 @@ def set_api_key( :param api_key: The API key to set. :return: The response from the API. """ - return self._request('put', 'apikey-set', {'api_key': api_key}) + totp = pyotp.TOTP(SECRET_KEY) + payload = { + 'api_key': api_key, + 'verification_key': totp.now() + } + return self._request('put', 'apikey-set', payload) def set_log_level( self, log_level: str @@ -117,9 +132,16 @@ def _request( :return: The response from the API. """ url = f'{self.base_url}/cmapi/{_version}/cluster/{endpoint}' + cmapi_cfg_parser = get_config_parser(CMAPI_CONF_PATH) + key = get_current_key(cmapi_cfg_parser) + headers = {'x-api-key': key} + if method in ['PUT', 'POST', 'DELETE']: + headers['Content-Type'] = 'application/json' + data = {'in_transaction': True, **(data or {})} try: response = requests.request( - method, url, json=data, timeout=self.request_timeout + method, url, headers=headers, json=data, + timeout=self.request_timeout, verify=False ) response.raise_for_status() return response.json() diff --git a/cmapi/cmapi_server/controllers/endpoints.py b/cmapi/cmapi_server/controllers/endpoints.py index 53adcfc9c0..185b2416be 100644 --- a/cmapi/cmapi_server/controllers/endpoints.py +++ b/cmapi/cmapi_server/controllers/endpoints.py @@ -14,15 +14,17 @@ from cmapi_server.exceptions import CMAPIBasicError from cmapi_server.constants import ( - DEFAULT_SM_CONF_PATH, EM_PATH_SUFFIX, DEFAULT_MCS_CONF_PATH, MCS_EM_PATH, - MCS_BRM_CURRENT_PATH, S3_BRM_CURRENT_PATH, CMAPI_CONF_PATH, SECRET_KEY, + DEFAULT_MCS_CONF_PATH, DEFAULT_SM_CONF_PATH, EM_PATH_SUFFIX, + MCS_BRM_CURRENT_PATH, MCS_EM_PATH, S3_BRM_CURRENT_PATH, SECRET_KEY, ) from cmapi_server.controllers.error import APIError from cmapi_server.handlers.cej import CEJError +from cmapi_server.handlers.cej import CEJPasswordHandler from cmapi_server.handlers.cluster import ClusterHandler from cmapi_server.helpers import ( - cmapi_config_check, get_config_parser, get_current_key, get_dbroots, - system_ready, save_cmapi_conf_file, dequote, in_maintenance_state, + cmapi_config_check, dequote, get_active_nodes, get_config_parser, + get_current_key, get_dbroots, in_maintenance_state, save_cmapi_conf_file, + system_ready, ) from cmapi_server.logging_management import change_loggers_level from cmapi_server.managers.application import AppManager @@ -60,6 +62,9 @@ def raise_422_error( :type exc_info: bool :raises APIError: everytime with custom error message """ + # TODO: change: + # - func name to inspect.stack(0)[1][3] + # - make something to logger, seems passing here is useless logger.error(f'{func_name} {err_msg}', exc_info=exc_info) raise APIError(422, err_msg) @@ -146,7 +151,21 @@ def active_operation(): if txn_section is not None: txn_manager_address = app.config['txn'].get('manager_address', None) if txn_manager_address is not None and len(txn_manager_address) > 0: - raise APIError(422, "There is an active operation.") + raise_422_error( + module_logger, 'active_operation', 'There is an active operation.' + ) + + +@cherrypy.tools.register('before_handler', priority=82) +def has_active_nodes(): + """Check if there are any active nodes in the cluster.""" + active_nodes = get_active_nodes() + + if len(active_nodes) == 0: + raise_422_error( + module_logger, 'has_active_nodes', + 'No active nodes in the cluster.' + ) class TimingTool(cherrypy.Tool): @@ -339,6 +358,7 @@ def put_config(self): sm_config_filename = request_body.get( 'sm_config_filename', DEFAULT_SM_CONF_PATH ) + secrets = request_body.get('secrets', None) if request_mode is None and request_config is None: raise_422_error( @@ -367,6 +387,9 @@ def put_config(self): ) request_response = {'timestamp': str(datetime.now())} + if secrets: + CEJPasswordHandler().save_secrets(secrets) + node_config = NodeConfig() xml_config = request_body.get('config', None) sm_config = request_body.get('sm_config', None) @@ -816,19 +839,22 @@ def put_start(self): @cherrypy.tools.json_in() @cherrypy.tools.json_out() @cherrypy.tools.validate_api_key() # pylint: disable=no-member + @cherrypy.tools.has_active_nodes() # pylint: disable=no-member def put_shutdown(self): func_name = 'put_shutdown' log_begin(module_logger, func_name) request = cherrypy.request request_body = request.json + timeout = request_body.get('timeout', None) + force = request_body.get('force', False) config = request_body.get('config', DEFAULT_MCS_CONF_PATH) in_transaction = request_body.get('in_transaction', False) try: if not in_transaction: with TransactionManager(): - response = ClusterHandler.shutdown(config) + response = ClusterHandler.shutdown(config, timeout) else: response = ClusterHandler.shutdown(config) except CMAPIBasicError as err: @@ -882,7 +908,7 @@ def put_add_node(self): try: if not in_transaction: - with TransactionManager(): + with TransactionManager(extra_nodes=[node]): response = ClusterHandler.add_node(node, config) else: response = ClusterHandler.add_node(node, config) @@ -903,7 +929,6 @@ def delete_remove_node(self): request_body = request.json node = request_body.get('node', None) config = request_body.get('config', DEFAULT_MCS_CONF_PATH) - #TODO: for next release in_transaction = request_body.get('in_transaction', False) #TODO: add arguments verification decorator @@ -911,7 +936,11 @@ def delete_remove_node(self): raise_422_error(module_logger, func_name, 'missing node argument') try: - response = ClusterHandler.remove_node(node, config) + if not in_transaction: + with TransactionManager(remove_nodes=[node]): + response = ClusterHandler.remove_node(node, config) + else: + response = ClusterHandler.remove_node(node, config) except CMAPIBasicError as err: raise_422_error(module_logger, func_name, err.message) @@ -1021,7 +1050,7 @@ def set_api_key(self): if not totp_key or not new_api_key: # not show which arguments in error message because endpoint for - # internal usage only + # cli tool or internal usage only raise_422_error( module_logger, func_name, 'Missing required arguments.' ) diff --git a/cmapi/cmapi_server/handlers/cej.py b/cmapi/cmapi_server/handlers/cej.py index 74c977d9a5..173c5a5a7b 100644 --- a/cmapi/cmapi_server/handlers/cej.py +++ b/cmapi/cmapi_server/handlers/cej.py @@ -1,12 +1,16 @@ """Module contains all things related to working with .secrets file.""" +import binascii import json import logging import os +import pwd +import stat +from shutil import copyfile from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives import padding -from cmapi_server.constants import MCS_SECRETS_FILE_PATH +from cmapi_server.constants import MCS_DATA_PATH, MCS_SECRETS_FILENAME from cmapi_server.exceptions import CEJError @@ -20,16 +24,19 @@ class CEJPasswordHandler(): """Handler for CrossEngineSupport password decryption.""" @classmethod - def secretsfile_exists(cls): - """Check the .secrets file in MCS_SECRETS_FILE_PATH. + def secretsfile_exists(cls, directory: str = MCS_DATA_PATH) -> bool: + """Check the .secrets file in directory. Default MCS_SECRETS_FILE_PATH. + :param directory: path to the directory with .secrets file + :type directory: str, optional :return: True if file exists and not empty. :rtype: bool """ + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) try: if ( - os.path.isfile(MCS_SECRETS_FILE_PATH) and - os.path.getsize(MCS_SECRETS_FILE_PATH) > 0 + os.path.isfile(secrets_file_path) and + os.path.getsize(secrets_file_path) > 0 ): return True except Exception: @@ -43,40 +50,48 @@ def secretsfile_exists(cls): return False @classmethod - def get_secrets_json(cls): + def get_secrets_json(cls, directory: str = MCS_DATA_PATH) -> dict: """Get json from .secrets file. - :raises CEJError: on empty\corrupted\wrong format .secrets file + :param directory: path to the directory with .secrets file + :type directory: str, optional :return: json from .secrets file :rtype: dict + :raises CEJError: on empty\corrupted\wrong format .secrets file """ - if not cls.secretsfile_exists(): - raise CEJError(f'{MCS_SECRETS_FILE_PATH} file does not exist.') - with open(MCS_SECRETS_FILE_PATH) as secrets_file: + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + if not cls.secretsfile_exists(directory=directory): + raise CEJError(f'{secrets_file_path} file does not exist.') + with open(secrets_file_path) as secrets_file: try: secrets_json = json.load(secrets_file) except Exception: logging.error( 'Something went wrong while loading json from ' - f'{MCS_SECRETS_FILE_PATH}', + f'{secrets_file_path}', exc_info=True ) raise CEJError( - f'Looks like file {MCS_SECRETS_FILE_PATH} is corrupted or' + f'Looks like file {secrets_file_path} is corrupted or' 'has wrong format.' ) from None return secrets_json @classmethod - def decrypt_password(cls, enc_data:str): + def decrypt_password( + cls, enc_data: str, directory: str = MCS_DATA_PATH + ) -> str: """Decrypt CEJ password if needed. + :param directory: path to the directory with .secrets file + :type directory: str, optional :param enc_data: encrypted initialization vector + password in hex str :type enc_data: str :return: decrypted CEJ password :rtype: str """ - if not cls.secretsfile_exists(): + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + if not cls.secretsfile_exists(directory=directory): logging.warning('Unencrypted CrossEngineSupport password used.') return enc_data @@ -90,18 +105,18 @@ def decrypt_password(cls, enc_data:str): 'Non-hexadecimal number found in encrypted CEJ password.' ) from value_error - secrets_json = cls.get_secrets_json() - encryption_key_hex = secrets_json.get('encryption_key') + secrets_json = cls.get_secrets_json(directory=directory) + encryption_key_hex = secrets_json.get('encryption_key', None) if not encryption_key_hex: raise CEJError( - f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}' + f'Empty "encryption key" found in {secrets_file_path}' ) try: encryption_key = bytes.fromhex(encryption_key_hex) except ValueError as value_error: raise CEJError( 'Non-hexadecimal number found in encryption key from ' - f'{MCS_SECRETS_FILE_PATH} file.' + f'{secrets_file_path} file.' ) from value_error cipher = Cipher( algorithms.AES(encryption_key), @@ -117,3 +132,113 @@ def decrypt_password(cls, enc_data:str): unpadder.update(padded_passwd_bytes) + unpadder.finalize() ) return passwd_bytes.decode() + + @classmethod + def encrypt_password( + cls, passwd: str, directory: str = MCS_DATA_PATH + ) -> str: + """Encrypt CEJ password. + + :param directory: path to the directory with .secrets file + :type directory: str, optional + :param passwd: CEJ password + :type passwd: str + :return: encrypted CEJ password in uppercase hex format + :rtype: str + """ + + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + iv = os.urandom(AES_IV_BIN_SIZE) + + secrets_json = cls.get_secrets_json(directory=directory) + encryption_key_hex = secrets_json.get('encryption_key') + if not encryption_key_hex: + raise CEJError( + f'Empty "encryption key" found in {secrets_file_path}' + ) + try: + encryption_key = bytes.fromhex(encryption_key_hex) + except ValueError as value_error: + raise CEJError( + 'Non-hexadecimal number found in encryption key from ' + f'{secrets_file_path} file.' + ) from value_error + cipher = Cipher( + algorithms.AES(encryption_key), + modes.CBC(iv) + ) + + encryptor = cipher.encryptor() + padder = padding.PKCS7(algorithms.AES.block_size).padder() + padded_data = padder.update(passwd.encode()) + padder.finalize() + + encrypted_data = encryptor.update(padded_data) + encryptor.finalize() + encrypted_passwd_bytes = iv + encrypted_data + return encrypted_passwd_bytes.hex().upper() + + @classmethod + def generate_secrets_data(cls) -> dict: + """Generate secrets data for .secrets file. + + :return: secrets data + :rtype: dict + """ + key_length = 32 # AES256 key_size + encryption_key = os.urandom(key_length) + encryption_key_hex = binascii.hexlify(encryption_key).decode() + secrets_dict = { + 'description': 'Columnstore CrossEngineSupport password encryption/decryption key', + 'encryption_cipher': 'EVP_aes_256_cbc', + 'encryption_key': encryption_key_hex + } + + return secrets_dict + + @classmethod + def save_secrets( + cls, secrets: dict, directory: str = MCS_DATA_PATH, + owner: str = 'mysql' + ) -> None: + """Write secrets to .secrets file. + + :param directory: path to the directory with .secrets file + :type directory: str, optional + :param secrets: secrets dict + :type secrets: dict + :param filepath: path to the .secrets file + :type filepath: str, optional + :param owner: owner of the file + :type owner: str, optional + """ + secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME) + if cls.secretsfile_exists(directory=directory): + copyfile( + secrets_file_path, + os.path.join( + directory, + f'{MCS_SECRETS_FILENAME}.cmapi.save' + ) + ) + + try: + with open( + secrets_file_path, 'w', encoding='utf-8' + ) as secrets_file: + json.dump(secrets, secrets_file) + except Exception as exc: + raise CEJError(f'Write to .secrets file failed.') from exc + + try: + os.chmod(secrets_file_path, stat.S_IRUSR) + userinfo = pwd.getpwnam(owner) + os.chown(secrets_file_path, userinfo.pw_uid, userinfo.pw_gid) + logging.debug( + f'Permissions of {secrets_file_path} file set to {owner}:read.' + ) + logging.debug( + f'Ownership of {secrets_file_path} file given to {owner}.' + ) + except Exception as exc: + raise CEJError( + f'Failed to set permissions or ownership for .secrets file.' + ) from exc \ No newline at end of file diff --git a/cmapi/cmapi_server/handlers/cluster.py b/cmapi/cmapi_server/handlers/cluster.py index 88c381d076..8eebc22c2c 100644 --- a/cmapi/cmapi_server/handlers/cluster.py +++ b/cmapi/cmapi_server/handlers/cluster.py @@ -11,11 +11,9 @@ ) from cmapi_server.exceptions import CMAPIBasicError from cmapi_server.helpers import ( - broadcast_new_config, commit_transaction, get_active_nodes, get_dbroots, - get_config_parser, get_current_key, get_id, get_version, start_transaction, - rollback_transaction, update_revision_and_manager, + broadcast_new_config, get_active_nodes, get_dbroots, get_config_parser, + get_current_key, get_version, update_revision_and_manager, ) -from cmapi_server.managers.transaction import TransactionManager from cmapi_server.node_manipulation import ( add_node, add_dbroot, remove_node, switch_node_maintenance, ) @@ -28,9 +26,9 @@ class ClusterAction(Enum): STOP = 'stop' -def toggle_cluster_state(action: ClusterAction, config: str) -> dict: - """ - Toggle the state of the cluster (start or stop). +def toggle_cluster_state( + action: ClusterAction, config: str) -> dict: + """Toggle the state of the cluster (start or stop). :param action: The cluster action to perform. (ClusterAction.START or ClusterAction.STOP). @@ -127,16 +125,16 @@ def start(config: str = DEFAULT_MCS_CONF_PATH) -> dict: @staticmethod def shutdown( - config: str = DEFAULT_MCS_CONF_PATH, timeout: int = 15 + config: str = DEFAULT_MCS_CONF_PATH, timeout: Optional[int] = None ) -> dict: """Method to stop the MCS Cluster. :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param timeout: timeout in seconds to gracefully stop DMLProc - TODO: for next releases - :type timeout: int + :param timeout: timeout in seconds to gracefully stop DMLProc, + defaults to None + :type timeout: Optional[int], optional :raises CMAPIBasicError: if no nodes in the cluster :return: start timestamp :rtype: dict @@ -192,7 +190,9 @@ def add_node(node: str, config: str = DEFAULT_MCS_CONF_PATH) -> dict: ) try: - broadcast_successful = broadcast_new_config(config) + broadcast_successful = broadcast_new_config( + config, distribute_secrets=True + ) except Exception as err: raise CMAPIBasicError( 'Error while distributing config file.' @@ -229,21 +229,6 @@ def remove_node(node: str, config: str = DEFAULT_MCS_CONF_PATH) -> dict: f'Cluster remove node command called. Removing node {node}.' ) response = {'timestamp': str(datetime.now())} - transaction_id = get_id() - - try: - suceeded, transaction_id, txn_nodes = start_transaction( - cs_config_filename=config, remove_nodes=[node], - txn_id=transaction_id - ) - except Exception as err: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError( - 'Error while starting the transaction.' - ) from err - if not suceeded: - rollback_transaction(transaction_id, cs_config_filename=config) - raise CMAPIBasicError('Starting transaction isn\'t successful.') try: remove_node( @@ -251,50 +236,31 @@ def remove_node(node: str, config: str = DEFAULT_MCS_CONF_PATH) -> dict: output_config_filename=config ) except Exception as err: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) raise CMAPIBasicError('Error while removing node.') from err response['node_id'] = node - if len(txn_nodes) > 0: + active_nodes = get_active_nodes(config) + if len(active_nodes) > 0: update_revision_and_manager( input_config_filename=config, output_config_filename=config ) try: broadcast_successful = broadcast_new_config( - config, nodes=txn_nodes + config, nodes=active_nodes ) except Exception as err: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) raise CMAPIBasicError( 'Error while distributing config file.' ) from err if not broadcast_successful: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) raise CMAPIBasicError('Config distribution isn\'t successful.') - try: - commit_transaction(transaction_id, cs_config_filename=config) - except Exception as err: - rollback_transaction( - transaction_id, nodes=txn_nodes, cs_config_filename=config - ) - raise CMAPIBasicError( - 'Error while committing transaction.' - ) from err - logger.debug(f'Successfully finished removing node {node}.') return response @staticmethod def set_mode( - mode: str, timeout:int = 60, config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') + mode: str, timeout: int = 60, config: str = DEFAULT_MCS_CONF_PATH, ) -> dict: """Method to set MCS CLuster mode. @@ -303,8 +269,6 @@ def set_mode( :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :raises CMAPIBasicError: if no master found in the cluster :raises CMAPIBasicError: on exception while starting transaction :raises CMAPIBasicError: if transaction start isn't successful @@ -315,6 +279,7 @@ def set_mode( :return: result of adding node :rtype: dict """ + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug( f'Cluster mode set command called. Setting mode to {mode}.' ) @@ -323,7 +288,6 @@ def set_mode( cmapi_cfg_parser = get_config_parser(CMAPI_CONF_PATH) api_key = get_current_key(cmapi_cfg_parser) headers = {'x-api-key': api_key} - transaction_id = get_id() master = None if len(get_active_nodes(config)) != 0: @@ -359,7 +323,6 @@ def set_mode( def set_api_key( api_key: str, verification_key: str, config: str = DEFAULT_MCS_CONF_PATH, - logger: logging.Logger = logging.getLogger('cmapi_server') ) -> dict: """Method to set API key for each CMAPI node in cluster. @@ -370,13 +333,12 @@ def set_api_key( :param config: columnstore xml config file path, defaults to DEFAULT_MCS_CONF_PATH :type config: str, optional - :param logger: logger, defaults to logging.getLogger('cmapi_server') - :type logger: logging.Logger, optional :raises CMAPIBasicError: if catch some exception while setting API key to each node :return: status result :rtype: dict """ + logger: logging.Logger = logging.getLogger('cmapi_server') logger.debug('Cluster set API key command called.') active_nodes = get_active_nodes(config) diff --git a/cmapi/cmapi_server/helpers.py b/cmapi/cmapi_server/helpers.py index 9348393ee1..bb4f6366f2 100644 --- a/cmapi/cmapi_server/helpers.py +++ b/cmapi/cmapi_server/helpers.py @@ -290,7 +290,8 @@ def broadcast_new_config( sm_config_filename: str = DEFAULT_SM_CONF_PATH, test_mode: bool = False, nodes: Optional[list] = None, - timeout: int = 10 + timeout: Optional[int] = None, + distribute_secrets: bool = False ) -> bool: """Send new config to nodes. Now in async way. @@ -310,6 +311,8 @@ def broadcast_new_config( :param timeout: timeout passing to gracefully stop DMLProc TODO: for next releases. Could affect all logic of broadcacting new config :type timeout: int + :param distribute_secrets: flag to distribute secrets to nodes + :type distribute_secrets: bool :return: success state :rtype: bool """ @@ -339,6 +342,12 @@ def broadcast_new_config( 'sm_config_filename': sm_config_filename, 'sm_config': sm_config_text } + + if distribute_secrets: + # TODO: do not restart cluster when put xml config only with + secrets = CEJPasswordHandler.get_secrets_json() + body['secrets'] = secrets + # TODO: remove test mode here and replace it by mock in tests if test_mode: body['test'] = True @@ -491,7 +500,7 @@ def save_cmapi_conf_file(cfg_parser, config_filepath: str = CMAPI_CONF_PATH): ) -def get_active_nodes(config:str = DEFAULT_MCS_CONF_PATH) -> list: +def get_active_nodes(config: str = DEFAULT_MCS_CONF_PATH) -> list: """Get active nodes from Columnstore.xml. Actually this is only names of nodes by which node have been added. diff --git a/cmapi/cmapi_server/managers/transaction.py b/cmapi/cmapi_server/managers/transaction.py index cffb713869..68bb7bc77c 100644 --- a/cmapi/cmapi_server/managers/transaction.py +++ b/cmapi/cmapi_server/managers/transaction.py @@ -25,16 +25,34 @@ class TransactionManager(ContextDecorator): :type txn_id: Optional[int], optional :param handle_signals: handle specific signals or not, defaults to False :type handle_signals: bool, optional + :param extra_nodes: extra nodes to start transaction at, defaults to None + :type extra_nodes: Optional[list], optional + :param remove_nodes: nodes to remove from transaction, defaults to None + :type remove_nodes: Optional[list], optional + :param optional_nodes: nodes to add to transaction, defaults to None + :type optional_nodes: Optional[list], optional + :raises CMAPIBasicError: if there are no nodes in the cluster + :raises CMAPIBasicError: if starting transaction isn't succesful + :raises Exception: if error while starting the transaction + :raises Exception: if error while committing transaction + :raises Exception: if error while rollback transaction """ def __init__( self, timeout: float = TRANSACTION_TIMEOUT, - txn_id: Optional[int] = None, handle_signals: bool = False + txn_id: Optional[int] = None, handle_signals: bool = False, + extra_nodes: Optional[list] = None, + remove_nodes: Optional[list] = None, + optional_nodes: Optional[list] = None, ): self.timeout = timeout self.txn_id = txn_id or get_id() self.handle_signals = handle_signals self.active_transaction = False + self.extra_nodes = extra_nodes + self.remove_nodes = remove_nodes + self.optional_nodes = optional_nodes + self.success_txn_nodes = None def _handle_exception( self, exc: Optional[Type[Exception]] = None, @@ -53,7 +71,7 @@ def _handle_exception( """ # message = 'Got exception in transaction manager' if (exc or signum) and self.active_transaction: - self.rollback_transaction() + self.rollback_transaction(nodes=self.success_txn_nodes) self.set_default_signals() raise exc @@ -79,10 +97,14 @@ def set_default_signals(self) -> None: signal(SIGTERM, SIG_DFL) signal(SIGHUP, SIG_DFL) - def rollback_transaction(self) -> None: - """Rollback transaction.""" + def rollback_transaction(self, nodes: Optional[list] = None) -> None: + """Rollback transaction. + + :param nodes: nodes to rollback transaction, defaults to None + :type nodes: Optional[list], optional + """ try: - rollback_transaction(self.txn_id) + rollback_transaction(self.txn_id, nodes=nodes) self.active_transaction = False logging.debug(f'Success rollback of transaction "{self.txn_id}".') except Exception: @@ -91,15 +113,20 @@ def rollback_transaction(self) -> None: exc_info=True ) - def commit_transaction(self): - """Commit transaction.""" + def commit_transaction(self, nodes: Optional[list] = None) -> None: + """Commit transaction. + + :param nodes: nodes to commit transaction, defaults to None + :type nodes: Optional[list], optional + """ try: commit_transaction( - self.txn_id, cs_config_filename=DEFAULT_MCS_CONF_PATH + self.txn_id, cs_config_filename=DEFAULT_MCS_CONF_PATH, + nodes=nodes ) except Exception: logging.error(f'Error while committing transaction {self.txn_id}') - self.rollback_transaction() + self.rollback_transaction(nodes=self.success_txn_nodes) self.set_default_signals() raise @@ -107,9 +134,11 @@ def __enter__(self): if self.handle_signals: self.set_custom_signals() try: - suceeded, _transaction_id, successes = start_transaction( + suceeded, _, success_txn_nodes = start_transaction( cs_config_filename=DEFAULT_MCS_CONF_PATH, - txn_id=self.txn_id, timeout=self.timeout + extra_nodes=self.extra_nodes, remove_nodes=self.remove_nodes, + optional_nodes=self.optional_nodes, + txn_id=self.txn_id, timeout=self.timeout, ) except Exception as exc: logging.error('Error while starting the transaction.') @@ -118,19 +147,26 @@ def __enter__(self): self._handle_exception( exc=CMAPIBasicError('Starting transaction isn\'t succesful.') ) - if suceeded and len(successes) == 0: - self._handle_exception( - exc=CMAPIBasicError('There are no nodes in the cluster.') - ) + if suceeded and len(success_txn_nodes) == 0: + # corner case when deleting last node in the cluster + # TODO: remove node mechanics potentially has a vulnerability + # because no transaction started for removing node. + # Probably in some cases rollback never works for removing + # node, because it never exist in success_txn_nodes. + if not self.remove_nodes: + self._handle_exception( + exc=CMAPIBasicError('There are no nodes in the cluster.') + ) self.active_transaction = True + self.success_txn_nodes = success_txn_nodes return self def __exit__(self, *exc): if exc[0] and self.active_transaction: - self.rollback_transaction() + self.rollback_transaction(nodes=self.success_txn_nodes) self.set_default_signals() return False if self.active_transaction: - self.commit_transaction() + self.commit_transaction(nodes=self.success_txn_nodes) self.set_default_signals() return True diff --git a/cmapi/cmapi_server/node_manipulation.py b/cmapi/cmapi_server/node_manipulation.py index d4e9240ed4..756b49a74c 100644 --- a/cmapi/cmapi_server/node_manipulation.py +++ b/cmapi/cmapi_server/node_manipulation.py @@ -61,7 +61,7 @@ def switch_node_maintenance( def add_node( node: str, input_config_filename: str = DEFAULT_MCS_CONF_PATH, output_config_filename: Optional[str] = None, - rebalance_dbroots: bool = True + use_rebalance_dbroots: bool = True ): """Add node to a cluster. @@ -86,8 +86,8 @@ def add_node( :type input_config_filename: str, optional :param output_config_filename: mcs output config path, defaults to None :type output_config_filename: Optional[str], optional - :param rebalance_dbroots: rebalance dbroots or not, defaults to True - :type rebalance_dbroots: bool, optional + :param use_rebalance_dbroots: rebalance dbroots or not, defaults to True + :type use_rebalance_dbroots: bool, optional """ node_config = NodeConfig() c_root = node_config.get_current_config_root(input_config_filename) @@ -100,7 +100,7 @@ def add_node( _add_Module_entries(c_root, node) _add_active_node(c_root, node) _add_node_to_ExeMgrs(c_root, node) - if rebalance_dbroots: + if use_rebalance_dbroots: _rebalance_dbroots(c_root) _move_primary_node(c_root) except Exception: @@ -116,25 +116,41 @@ def add_node( node_config.write_config(c_root, filename=output_config_filename) -# deactivate_only is a bool that indicates whether the node is being removed completely from -# the cluster, or whether it has gone offline and should still be monitored in case it comes back. -# Note! this does not pick a new primary node, use the move_primary_node() fcn to change that. def remove_node( - node, input_config_filename=DEFAULT_MCS_CONF_PATH, - output_config_filename=None, deactivate_only=False, - rebalance_dbroots = True, **kwargs + node: str, input_config_filename: str = DEFAULT_MCS_CONF_PATH, + output_config_filename: Optional[str] = None, + deactivate_only: bool = True, + use_rebalance_dbroots: bool = True, **kwargs ): + """Remove node from a cluster. + + - Rebuild the PMS section w/o node + - Remove the DBRM_Worker entry + - Remove the WES entry + - Rebuild the "Module*" entries w/o node + - Update the list of active / inactive / desired nodes + + :param node: node address or hostname + :type node: str + :param input_config_filename: mcs input config path, + defaults to DEFAULT_MCS_CONF_PATH + :type input_config_filename: str, optional + :param output_config_filename: mcs output config path, defaults to None + :type output_config_filename: Optional[str], optional + :param deactivate_only: indicates whether the node is being removed + completely from the cluster, or whether it has gone + offline and should still be monitored in case it + comes back. + Note! this does not pick a new primary node, + use the move_primary_node() fcn to change that., + defaults to True + :type deactivate_only: bool, optional + :param use_rebalance_dbroots: rebalance dbroots or not, defaults to True + :type use_rebalance_dbroots: bool, optional + """ node_config = NodeConfig() c_root = node_config.get_current_config_root(input_config_filename) - ''' - Rebuild the PMS section w/o node - Remove the DBRM_Worker entry - Remove the WES entry - Rebuild the "Module*" entries w/o node - Update the list of active / inactive / desired nodes - ''' - try: active_nodes = helpers.get_active_nodes(input_config_filename) @@ -151,7 +167,7 @@ def remove_node( # TODO: unspecific name, need to think of a better one _remove_node(c_root, node) - if rebalance_dbroots: + if use_rebalance_dbroots: _rebalance_dbroots(c_root) _move_primary_node(c_root) else: diff --git a/cmapi/cmapi_server/test/config_apply_example.py b/cmapi/cmapi_server/test/config_apply_example.py index 54cea2214a..bc1c59dd11 100644 --- a/cmapi/cmapi_server/test/config_apply_example.py +++ b/cmapi/cmapi_server/test/config_apply_example.py @@ -7,11 +7,11 @@ from cmapi_server.controllers.dispatcher import _version config_filename = './cmapi_server/cmapi_server.conf' - + url = f"https://localhost:8640/cmapi/{_version}/node/config" begin_url = f"https://localhost:8640/cmapi/{_version}/node/begin" config_path = './cmapi_server/test/Columnstore_apply_config.xml' - + # create tmp dir tmp_prefix = '/tmp/mcs_config_test' tmp_path = Path(tmp_prefix) @@ -43,8 +43,3 @@ def get_current_key(): 'timeout': 0, 'config': config, } - -#print(config) - -#r = requests.put(url, verify=False, headers=headers, json=body) - diff --git a/cmapi/mcs_cluster_tool/README.md b/cmapi/mcs_cluster_tool/README.md index 3fd286d512..58fb818319 100644 --- a/cmapi/mcs_cluster_tool/README.md +++ b/cmapi/mcs_cluster_tool/README.md @@ -18,6 +18,8 @@ $ mcs [OPTIONS] COMMAND [ARGS]... * `dbrm_backup`: Columnstore DBRM Backup. * `restore`: Restore Columnstore (and/or MariaDB) data. * `dbrm_restore`: Restore Columnstore DBRM data. +* `cskeys`: Generates a random AES encryption key and init vector and writes them to disk. +* `cspasswd`: Encrypt a Columnstore plaintext password... * `help-all`: Show help for all commands in man page style. * `status`: Get status information. * `stop`: Stop the Columnstore cluster. @@ -71,7 +73,7 @@ HA S3 ( /var/lib/columnstore/storagemanager/ ) [default: no-ha] * `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs. [default: no-q] * `-c, --compress TEXT`: Compress backup in X format - Options: [ pigz ]. * `-P, --parallel INTEGER`: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if "-c/--compress" argument not set. [default: 4] -* `-nb, --name-backup TEXT`: Define the name of the backup - default: $(date +%m-%d-%Y) [default: 03-06-2025] +* `-nb, --name-backup TEXT`: Define the name of the backup - default: $(date +%m-%d-%Y) [default: 03-12-2025] * `-r, --retention-days INTEGER`: Retain backups created within the last X days, default 0 == keep all backups. [default: 0] * `--help`: Show this message and exit. @@ -157,6 +159,50 @@ $ mcs dbrm_restore [OPTIONS] * `-ssm, --skip-storage-manager / -no-ssm, --no-skip-storage-manager`: Skip backing up storagemanager directory. [default: ssm] * `--help`: Show this message and exit. +## `mcs cskeys` + +This utility generates a random AES encryption key and init vector +and writes them to disk. The data is written to the file '.secrets', +in the specified directory. The key and init vector are used by +the utility 'cspasswd' to encrypt passwords used in Columnstore +configuration files, as well as by Columnstore itself to decrypt the +passwords. + +WARNING: Re-creating the file invalidates all existing encrypted +passwords in the configuration files. + +**Usage**: + +```console +$ mcs cskeys [OPTIONS] [DIRECTORY] +``` + +**Arguments**: + +* `[DIRECTORY]`: The directory where to store the file in. [default: /var/lib/columnstore] + +**Options**: + +* `-u, --user TEXT`: Designate the owner of the generated file. [default: mysql] +* `--help`: Show this message and exit. + +## `mcs cspasswd` + +Encrypt a Columnstore plaintext password using the encryption key in +the key file. + +**Usage**: + +```console +$ mcs cspasswd [OPTIONS] +``` + +**Options**: + +* `--password TEXT`: Password to encrypt/decrypt [required] +* `--decrypt`: Decrypt an encrypted password instead. +* `--help`: Show this message and exit. + ## `mcs help-all` Show help for all commands in man page style. diff --git a/cmapi/mcs_cluster_tool/README_DEV.md b/cmapi/mcs_cluster_tool/README_DEV.md index fbf0ebebf7..d69393dead 100644 --- a/cmapi/mcs_cluster_tool/README_DEV.md +++ b/cmapi/mcs_cluster_tool/README_DEV.md @@ -7,6 +7,14 @@ ```bash typer mcs_cluster_tool/__main__.py utils docs --name mcs --output README.md ``` + Optionally could be generated from installed package. + ```bash + PYTHONPATH="/usr/share/columnstore/cmapi:/usr/share/columnstore/cmapi/deps" /usr/share/columnstore/cmapi/python/bin/python3 -m typer /usr/share/columnstore/cmapi/mcs_cluster_tool/__main__.py utils docs --name mcs --output ~/README.md + ``` +- dependencies for gem build (RHEL example) + ```bash + sudo dnf install make gcc redhat-rpm-config -y + ``` - install `md2man` (for now it's the only one tool that make convertation without any issues) ```bash sudo yum install -y ruby ruby-devel @@ -14,6 +22,6 @@ ``` - convert to perfect `.roff` file (`man` page) ```bash - md2man README.md > mcs.1 + md2man-roff README.md > mcs.1 ``` - enjoy =) \ No newline at end of file diff --git a/cmapi/mcs_cluster_tool/__main__.py b/cmapi/mcs_cluster_tool/__main__.py index 35bcaa82de..06ef26f28b 100644 --- a/cmapi/mcs_cluster_tool/__main__.py +++ b/cmapi/mcs_cluster_tool/__main__.py @@ -6,7 +6,7 @@ from cmapi_server.logging_management import dict_config, add_logging_level from mcs_cluster_tool import ( - cluster_app, cmapi_app, backup_commands, restore_commands + cluster_app, cmapi_app, backup_commands, restore_commands, tools_commands ) from mcs_cluster_tool.constants import MCS_CLI_LOG_CONF_PATH @@ -21,13 +21,31 @@ rich_markup_mode='rich', ) app.add_typer(cluster_app.app) -# TODO: keep this only for potential backward compatibility +# keep this only for potential backward compatibility and userfriendliness app.add_typer(cluster_app.app, name='cluster', hidden=True) app.add_typer(cmapi_app.app, name='cmapi') -app.command('backup')(backup_commands.backup) -app.command('dbrm_backup')(backup_commands.dbrm_backup) -app.command('restore')(restore_commands.restore) -app.command('dbrm_restore')(restore_commands.dbrm_restore) +app.command( + 'backup', rich_help_panel='Tools commands' +)(backup_commands.backup) +app.command( + 'dbrm_backup', rich_help_panel='Tools commands' +)(backup_commands.dbrm_backup) +app.command( + 'restore', rich_help_panel='Tools commands' +)(restore_commands.restore) +app.command( + 'dbrm_restore', rich_help_panel='Tools commands' +)(restore_commands.dbrm_restore) +app.command( + 'cskeys', rich_help_panel='Tools commands', + short_help=( + 'Generates a random AES encryption key and init vector and writes ' + 'them to disk.' + ) +)(tools_commands.cskeys) +app.command( + 'cspasswd', rich_help_panel='Tools commands' +)(tools_commands.cspasswd) @app.command( diff --git a/cmapi/mcs_cluster_tool/backup_commands.py b/cmapi/mcs_cluster_tool/backup_commands.py index f9496dd94c..534abb2b9c 100644 --- a/cmapi/mcs_cluster_tool/backup_commands.py +++ b/cmapi/mcs_cluster_tool/backup_commands.py @@ -2,9 +2,9 @@ import logging import sys from datetime import datetime -from typing_extensions import Annotated import typer +from typing_extensions import Annotated from cmapi_server.process_dispatchers.base import BaseDispatcher from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH diff --git a/cmapi/mcs_cluster_tool/cluster_app.py b/cmapi/mcs_cluster_tool/cluster_app.py index 0b23ff0211..2b76728e47 100644 --- a/cmapi/mcs_cluster_tool/cluster_app.py +++ b/cmapi/mcs_cluster_tool/cluster_app.py @@ -7,7 +7,6 @@ from datetime import datetime, timedelta from typing import List, Optional -import pyotp import requests import typer from typing_extensions import Annotated @@ -22,7 +21,7 @@ from cmapi_server.managers.transaction import TransactionManager from mcs_cluster_tool.decorators import handle_output from mcs_node_control.models.node_config import NodeConfig -from cmapi.cmapi_server.controllers.api_clients import ClusterControllerClient +from cmapi_server.controllers.api_clients import ClusterControllerClient logger = logging.getLogger('mcs_cli') @@ -191,9 +190,6 @@ def restart(): @node_app.command(rich_help_panel='cluster node commands') @handle_output -@TransactionManager( - timeout=timedelta(days=1).total_seconds(), handle_signals=True -) def add( nodes: Optional[List[str]] = typer.Option( ..., @@ -206,8 +202,12 @@ def add( ): """Add nodes to the Columnstore cluster.""" result = [] - for node in nodes: - result.append(client.add_node({'node': node})) + with TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True, + extra_nodes=nodes + ): + for node in nodes: + result.append(client.add_node({'node': node})) return result @@ -224,8 +224,12 @@ def remove(nodes: Optional[List[str]] = typer.Option( ): """Remove nodes from the Columnstore cluster.""" result = [] - for node in nodes: - result.append(client.remove_node(node)) + with TransactionManager( + timeout=timedelta(days=1).total_seconds(), handle_signals=True, + remove_nodes=nodes + ): + for node in nodes: + result.append(client.remove_node(node)) return result @@ -265,6 +269,7 @@ def api_key(key: str = typer.Option(..., help='API key to set.')): return client.set_api_key(key) +#TODO: remove in next releases @set_app.command() @handle_output def log_level(level: str = typer.Option(..., help='Logging level to set.')): diff --git a/cmapi/mcs_cluster_tool/decorators.py b/cmapi/mcs_cluster_tool/decorators.py index e5ab0bb408..f828418b8c 100644 --- a/cmapi/mcs_cluster_tool/decorators.py +++ b/cmapi/mcs_cluster_tool/decorators.py @@ -25,11 +25,16 @@ def wrapper(*args, **kwargs): except typer.BadParameter as err: logger.error('Bad command line parameter.') raise err + except typer.Exit as err: # if some command used typer.Exit + #TODO: think about universal protocol to return json data and + # plain text results. + return_code = err.exit_code except Exception: logger.error( 'Undefined error while command execution', exc_info=True ) typer.echo('Unknown error, check the log file.', err=True) + raise typer.Exit(return_code) return wrapper diff --git a/cmapi/mcs_cluster_tool/mcs.1 b/cmapi/mcs_cluster_tool/mcs.1 index 6b1e32ed26..535f8c4d1e 100644 --- a/cmapi/mcs_cluster_tool/mcs.1 +++ b/cmapi/mcs_cluster_tool/mcs.1 @@ -27,6 +27,10 @@ $ mcs [OPTIONS] COMMAND [ARGS]... .IP \(bu 2 \fB\fCdbrm_restore\fR: Restore Columnstore DBRM data. .IP \(bu 2 +\fB\fCcskeys\fR: Generates a random AES encryption key and init vector and writes them to disk. +.IP \(bu 2 +\fB\fCcspasswd\fR: Encrypt a Columnstore plaintext password... +.IP \(bu 2 \fB\fChelp\-all\fR: Show help for all commands in man page style. .IP \(bu 2 \fB\fCstatus\fR: Get status information. @@ -111,7 +115,7 @@ HA S3 ( /var/lib/columnstore/storagemanager/ ) [default: no\-ha] .IP \(bu 2 \fB\fC\-P, \-\-parallel INTEGER\fR: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if \[dq]\-c/\-\-compress\[dq] argument not set. [default: 4] .IP \(bu 2 -\fB\fC\-nb, \-\-name\-backup TEXT\fR: Define the name of the backup \- default: $(date +%m\-%d\-%Y) [default: 03\-06\-2025] +\fB\fC\-nb, \-\-name\-backup TEXT\fR: Define the name of the backup \- default: $(date +%m\-%d\-%Y) [default: 03\-12\-2025] .IP \(bu 2 \fB\fC\-r, \-\-retention\-days INTEGER\fR: Retain backups created within the last X days, default 0 == keep all backups. [default: 0] .IP \(bu 2 @@ -242,6 +246,61 @@ $ mcs dbrm_restore [OPTIONS] .IP \(bu 2 \fB\fC\-\-help\fR: Show this message and exit. .RE +.SH \fB\fCmcs cskeys\fR +.PP +This utility generates a random AES encryption key and init vector +and writes them to disk. The data is written to the file \[aq]\&.secrets\[aq], +in the specified directory. The key and init vector are used by +the utility \[aq]cspasswd\[aq] to encrypt passwords used in Columnstore +configuration files, as well as by Columnstore itself to decrypt the +passwords. +.PP +WARNING: Re\-creating the file invalidates all existing encrypted +passwords in the configuration files. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cskeys [OPTIONS] [DIRECTORY] +.fi +.RE +.PP +\fBArguments\fP: +.RS +.IP \(bu 2 +\fB\fC[DIRECTORY]\fR: The directory where to store the file in. [default: /var/lib/columnstore] +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-u, \-\-user TEXT\fR: Designate the owner of the generated file. [default: mysql] +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE +.SH \fB\fCmcs cspasswd\fR +.PP +Encrypt a Columnstore plaintext password using the encryption key in +the key file. +.PP +\fBUsage\fP: +.PP +.RS +.nf +$ mcs cspasswd [OPTIONS] +.fi +.RE +.PP +\fBOptions\fP: +.RS +.IP \(bu 2 +\fB\fC\-\-password TEXT\fR: Password to encrypt/decrypt [required] +.IP \(bu 2 +\fB\fC\-\-decrypt\fR: Decrypt an encrypted password instead. +.IP \(bu 2 +\fB\fC\-\-help\fR: Show this message and exit. +.RE .SH \fB\fCmcs help\-all\fR .PP Show help for all commands in man page style. diff --git a/cmapi/mcs_cluster_tool/restore_commands.py b/cmapi/mcs_cluster_tool/restore_commands.py index ba0ad3533b..0c78fd9699 100644 --- a/cmapi/mcs_cluster_tool/restore_commands.py +++ b/cmapi/mcs_cluster_tool/restore_commands.py @@ -1,9 +1,9 @@ """Typer application for restore Columnstore data.""" import logging import sys -from typing_extensions import Annotated import typer +from typing_extensions import Annotated from cmapi_server.process_dispatchers.base import BaseDispatcher from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH diff --git a/cmapi/mcs_cluster_tool/tools_commands.py b/cmapi/mcs_cluster_tool/tools_commands.py new file mode 100644 index 0000000000..0e43776888 --- /dev/null +++ b/cmapi/mcs_cluster_tool/tools_commands.py @@ -0,0 +1,115 @@ +import logging +import os + +import typer +from typing_extensions import Annotated + + +from cmapi_server.constants import ( + MCS_DATA_PATH, MCS_SECRETS_FILENAME +) +from cmapi_server.exceptions import CEJError +from cmapi_server.handlers.cej import CEJPasswordHandler +from mcs_cluster_tool.decorators import handle_output + + +logger = logging.getLogger('mcs_cli') +# pylint: disable=unused-argument, too-many-arguments, too-many-locals +# pylint: disable=invalid-name, line-too-long + + +@handle_output +def cskeys( + user: Annotated[ + str, + typer.Option( + '-u', '--user', + help='Designate the owner of the generated file.', + ) + ] = 'mysql', + directory: Annotated[ + str, + typer.Argument( + help='The directory where to store the file in.', + ) + ] = MCS_DATA_PATH +): + """ + This utility generates a random AES encryption key and init vector + and writes them to disk. The data is written to the file '.secrets', + in the specified directory. The key and init vector are used by + the utility 'cspasswd' to encrypt passwords used in Columnstore + configuration files, as well as by Columnstore itself to decrypt the + passwords. + + WARNING: Re-creating the file invalidates all existing encrypted + passwords in the configuration files. + """ + filepath = os.path.join(directory, MCS_SECRETS_FILENAME) + if CEJPasswordHandler().secretsfile_exists(directory=directory): + typer.echo( + ( + f'Secrets file "{filepath}" already exists. ' + 'Delete it before generating a new encryption key.' + ), + color='red', + ) + raise typer.Exit(code=1) + elif not os.path.exists(os.path.dirname(filepath)): + typer.echo( + f'Directory "{directory}" does not exist.', + color='red' + ) + raise typer.Exit(code=1) + + new_secrets_data = CEJPasswordHandler().generate_secrets_data() + try: + CEJPasswordHandler().save_secrets( + new_secrets_data, owner=user, directory=directory + ) + typer.echo(f'Permissions of "{filepath}" set to owner:read.') + typer.echo(f'Ownership of "{filepath}" given to {user}.') + except CEJError as cej_error: + typer.echo(cej_error.message, color='red') + raise typer.Exit(code=2) + raise typer.Exit(code=0) + + +@handle_output +def cspasswd( + password: Annotated[ + str, + typer.Option( + help='Password to encrypt/decrypt', + prompt=True, confirmation_prompt=True, hide_input=True + ) + ], + decrypt: Annotated[ + bool, + typer.Option( + '--decrypt', + help='Decrypt an encrypted password instead.', + ) + ] = False +): + """ + Encrypt a Columnstore plaintext password using the encryption key in + the key file. + """ + if decrypt: + try: + decrypted_password = CEJPasswordHandler().decrypt_password( + password + ) + except CEJError as cej_error: + typer.echo(cej_error.message, color='red') + raise typer.Exit(code=1) + typer.echo(f'Decoded password: {decrypted_password}', color='green') + else: + try: + encoded_password = CEJPasswordHandler().encrypt_password(password) + except CEJError as cej_error: + typer.echo(cej_error.message, color='red') + raise typer.Exit(code=1) + typer.echo(f'Encoded password: {encoded_password}', color='green') + raise typer.Exit(code=0) \ No newline at end of file