MCOL-5019: review fixes.

Author: mariadb-AlanMologorsky

cmapi/cmapi_server/handlers/cej.py

@@ -10,9 +10,7 @@
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.primitives import padding
 
-from cmapi_server.constants import (
-    MCS_DATA_PATH, MCS_SECRETS_FILENAME, MCS_SECRETS_FILE_PATH
-)
+from cmapi_server.constants import MCS_DATA_PATH, MCS_SECRETS_FILENAME
 from cmapi_server.exceptions import CEJError
 
 
@@ -26,16 +24,19 @@ class CEJPasswordHandler():
     """Handler for CrossEngineSupport password decryption."""
 
     @classmethod
-    def secretsfile_exists(cls) -> bool:
-        """Check the .secrets file in MCS_SECRETS_FILE_PATH.
+    def secretsfile_exists(cls, directory: str = MCS_DATA_PATH) -> bool:
+        """Check the .secrets file in directory. Default MCS_SECRETS_FILE_PATH.
 
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :return: True if file exists and not empty.
         :rtype: bool
         """
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
         try:
             if (
-                os.path.isfile(MCS_SECRETS_FILE_PATH) and
-                os.path.getsize(MCS_SECRETS_FILE_PATH) > 0
+                os.path.isfile(secrets_file_path) and
+                os.path.getsize(secrets_file_path) > 0
             ):
                 return True
         except Exception:
@@ -49,40 +50,48 @@ def secretsfile_exists(cls) -> bool:
         return False
 
     @classmethod
-    def get_secrets_json(cls) -> dict:
+    def get_secrets_json(cls, directory: str = MCS_DATA_PATH) -> dict:
         """Get json from .secrets file.
 
-        :raises CEJError: on empty\corrupted\wrong format .secrets file
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :return: json from .secrets file
         :rtype: dict
+        :raises CEJError: on empty\corrupted\wrong format .secrets file
         """
-        if not cls.secretsfile_exists():
-            raise CEJError(f'{MCS_SECRETS_FILE_PATH} file does not exist.')
-        with open(MCS_SECRETS_FILE_PATH) as secrets_file:
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
+        if not cls.secretsfile_exists(directory=directory):
+            raise CEJError(f'{secrets_file_path} file does not exist.')
+        with open(secrets_file_path) as secrets_file:
             try:
                 secrets_json = json.load(secrets_file)
             except Exception:
                 logging.error(
                     'Something went wrong while loading json from '
-                    f'{MCS_SECRETS_FILE_PATH}',
+                    f'{secrets_file_path}',
                     exc_info=True
                 )
                 raise CEJError(
-                    f'Looks like file {MCS_SECRETS_FILE_PATH} is corrupted or'
+                    f'Looks like file {secrets_file_path} is corrupted or'
                     'has wrong format.'
                 ) from None
             return secrets_json
 
     @classmethod
-    def decrypt_password(cls, enc_data: str) -> str:
+    def decrypt_password(
+        cls, enc_data: str, directory: str = MCS_DATA_PATH
+    ) -> str:
         """Decrypt CEJ password if needed.
 
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :param enc_data: encrypted initialization vector + password in hex str
         :type enc_data: str
         :return: decrypted CEJ password
         :rtype: str
         """
-        if not cls.secretsfile_exists():
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
+        if not cls.secretsfile_exists(directory=directory):
             logging.warning('Unencrypted CrossEngineSupport password used.')
             return enc_data
 
@@ -96,18 +105,18 @@ def decrypt_password(cls, enc_data: str) -> str:
                 'Non-hexadecimal number found in encrypted CEJ password.'
             ) from value_error
 
-        secrets_json = cls.get_secrets_json()
+        secrets_json = cls.get_secrets_json(directory=directory)
         encryption_key_hex = secrets_json.get('encryption_key', None)
         if not encryption_key_hex:
             raise CEJError(
-                f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
+                f'Empty "encryption key" found in {secrets_file_path}'
             )
         try:
             encryption_key = bytes.fromhex(encryption_key_hex)
         except ValueError as value_error:
             raise CEJError(
                 'Non-hexadecimal number found in encryption key from '
-                f'{MCS_SECRETS_FILE_PATH} file.'
+                f'{secrets_file_path} file.'
             ) from value_error
         cipher = Cipher(
             algorithms.AES(encryption_key),
@@ -125,21 +134,34 @@ def decrypt_password(cls, enc_data: str) -> str:
         return passwd_bytes.decode()
 
     @classmethod
-    def encrypt_password(cls, passwd: str) -> str:
-        iv = os.urandom(size=AES_IV_BIN_SIZE)
+    def encrypt_password(
+        cls, passwd: str, directory: str = MCS_DATA_PATH
+    ) -> str:
+        """Encrypt CEJ password.
+
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
+        :param passwd: CEJ password
+        :type passwd: str
+        :return: encrypted CEJ password in uppercase hex format
+        :rtype: str
+        """
 
-        secrets_json = cls.get_secrets_json()
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
+        iv = os.urandom(AES_IV_BIN_SIZE)
+
+        secrets_json = cls.get_secrets_json(directory=directory)
         encryption_key_hex = secrets_json.get('encryption_key')
         if not encryption_key_hex:
             raise CEJError(
-                f'Empty "encryption key" found in {MCS_SECRETS_FILE_PATH}'
+                f'Empty "encryption key" found in {secrets_file_path}'
             )
         try:
             encryption_key = bytes.fromhex(encryption_key_hex)
         except ValueError as value_error:
             raise CEJError(
                 'Non-hexadecimal number found in encryption key from '
-                f'{MCS_SECRETS_FILE_PATH} file.'
+                f'{secrets_file_path} file.'
             ) from value_error
         cipher = Cipher(
             algorithms.AES(encryption_key),
@@ -151,8 +173,8 @@ def encrypt_password(cls, passwd: str) -> str:
         padded_data = padder.update(passwd.encode()) + padder.finalize()
 
         encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
-
-        return iv + encrypted_data
+        encrypted_passwd_bytes = iv + encrypted_data
+        return encrypted_passwd_bytes.hex().upper()
 
     @classmethod
     def generate_secrets_data(cls) -> dict:
@@ -161,8 +183,8 @@ def generate_secrets_data(cls) -> dict:
         :return: secrets data
         :rtype: dict
         """
-        key_length = algorithms.AES256.key_size // 8
-        encryption_key = os.urandom(size=key_length)
+        key_length = 32  # AES256 key_size
+        encryption_key = os.urandom(key_length)
         encryption_key_hex = binascii.hexlify(encryption_key).decode()
         secrets_dict = {
             'description': 'Columnstore CrossEngineSupport password encryption/decryption key',
@@ -174,41 +196,48 @@ def generate_secrets_data(cls) -> dict:
 
     @classmethod
     def save_secrets(
-        cls, secrets: dict, filepath: str = MCS_SECRETS_FILE_PATH,
+        cls, secrets: dict, directory: str = MCS_DATA_PATH,
         owner: str = 'mysql'
     ) -> None:
         """Write secrets to .secrets file.
 
+        :param directory: path to the directory with .secrets file
+        :type directory: str, optional
         :param secrets: secrets dict
         :type secrets: dict
         :param filepath: path to the .secrets file
         :type filepath: str, optional
         :param owner: owner of the file
         :type owner: str, optional
         """
-        if cls.secretsfile_exists():
+        secrets_file_path = os.path.join(directory, MCS_SECRETS_FILENAME)
+        if cls.secretsfile_exists(directory=directory):
             copyfile(
-                filepath,
+                secrets_file_path,
                 os.path.join(
-                    os.path.dirname(filepath),
-                    f'{os.path.basename(filepath)}.cmapi.save'
+                    directory,
+                    f'{MCS_SECRETS_FILENAME}.cmapi.save'
                 )
             )
 
         try:
             with open(
-                MCS_SECRETS_FILE_PATH, 'w', encoding='utf-8'
+                secrets_file_path, 'w', encoding='utf-8'
             ) as secrets_file:
                 json.dump(secrets, secrets_file)
         except Exception as exc:
             raise CEJError(f'Write to .secrets file failed.') from exc
 
         try:
-            os.chmod(MCS_SECRETS_FILE_PATH, stat.S_IRUSR)
+            os.chmod(secrets_file_path, stat.S_IRUSR)
             userinfo = pwd.getpwnam(owner)
-            os.chown(MCS_SECRETS_FILE_PATH, userinfo.pw_uid, userinfo.pw_gid)
-            logging.debug(f'Permissions of .secrets file set to {owner}:read.')
-            logging.debug(f'Ownership of .secrets file given to {owner}.')
+            os.chown(secrets_file_path, userinfo.pw_uid, userinfo.pw_gid)
+            logging.debug(
+                f'Permissions of {secrets_file_path} file set to {owner}:read.'
+            )
+            logging.debug(
+                f'Ownership of {secrets_file_path} file given to {owner}.'
+            )
         except Exception as exc:
             raise CEJError(
                 f'Failed to set permissions or ownership for .secrets file.'

cmapi/mcs_cluster_tool/README.md

@@ -18,6 +18,8 @@ $ mcs [OPTIONS] COMMAND [ARGS]...
 * `dbrm_backup`: Columnstore DBRM Backup.
 * `restore`: Restore Columnstore (and/or MariaDB) data.
 * `dbrm_restore`: Restore Columnstore DBRM data.
+* `cskeys`: Generates a random AES encryption key and init vector and writes them to disk.
+* `cspasswd`: Encrypt a Columnstore plaintext password...
 * `help-all`: Show help for all commands in man page style.
 * `status`: Get status information.
 * `stop`: Stop the Columnstore cluster.
@@ -71,7 +73,7 @@ HA S3           ( /var/lib/columnstore/storagemanager/ )  [default: no-ha]
 * `-q, --quiet / -no-q, --no-quiet`: Silence verbose copy command outputs.  [default: no-q]
 * `-c, --compress TEXT`: Compress backup in X format - Options: [ pigz ].
 * `-P, --parallel INTEGER`: Determines if columnstore data directories will have multiple rsync running at the same time for different subfolders to parallelize writes. Ignored if "-c/--compress" argument not set.  [default: 4]
-* `-nb, --name-backup TEXT`: Define the name of the backup - default: $(date +%m-%d-%Y)  [default: 03-06-2025]
+* `-nb, --name-backup TEXT`: Define the name of the backup - default: $(date +%m-%d-%Y)  [default: 03-12-2025]
 * `-r, --retention-days INTEGER`: Retain backups created within the last X days, default 0 == keep all backups.  [default: 0]
 * `--help`: Show this message and exit.
 
@@ -157,6 +159,50 @@ $ mcs dbrm_restore [OPTIONS]
 * `-ssm, --skip-storage-manager / -no-ssm, --no-skip-storage-manager`: Skip backing up storagemanager directory.  [default: ssm]
 * `--help`: Show this message and exit.
 
+## `mcs cskeys`
+
+This utility generates a random AES encryption key and init vector
+and writes them to disk. The data is written to the file '.secrets',
+in the specified directory. The key and init vector are used by
+the utility 'cspasswd' to encrypt passwords used in Columnstore
+configuration files, as well as by Columnstore itself to decrypt the
+passwords.
+
+WARNING: Re-creating the file invalidates all existing encrypted
+passwords in the configuration files.
+
+**Usage**:
+
+```console
+$ mcs cskeys [OPTIONS] [DIRECTORY]
+```
+
+**Arguments**:
+
+* `[DIRECTORY]`: The directory where to store the file in.  [default: /var/lib/columnstore]
+
+**Options**:
+
+* `-u, --user TEXT`: Designate the owner of the generated file.  [default: mysql]
+* `--help`: Show this message and exit.
+
+## `mcs cspasswd`
+
+Encrypt a Columnstore plaintext password using the encryption key in
+the key file.
+
+**Usage**:
+
+```console
+$ mcs cspasswd [OPTIONS]
+```
+
+**Options**:
+
+* `--password TEXT`: Password to encrypt/decrypt  [required]
+* `--decrypt`: Decrypt an encrypted password instead.
+* `--help`: Show this message and exit.
+
 ## `mcs help-all`
 
 Show help for all commands in man page style.

cmapi/mcs_cluster_tool/README_DEV.md

@@ -7,13 +7,21 @@
     ```bash
     typer mcs_cluster_tool/__main__.py utils docs --name mcs --output README.md
     ```
+    Optionally could be generated from installed package.
+    ```bash
+    PYTHONPATH="/usr/share/columnstore/cmapi:/usr/share/columnstore/cmapi/deps" /usr/share/columnstore/cmapi/python/bin/python3 -m  typer /usr/share/columnstore/cmapi/mcs_cluster_tool/__main__.py utils docs --name mcs --output ~/README.md
+    ```
+- dependencies for gem build (RHEL example)
+    ```bash
+    sudo dnf install make gcc redhat-rpm-config -y
+    ```
 - install `md2man` (for now it's the only one tool that make convertation without any issues)
     ```bash
     sudo yum install -y ruby ruby-devel
     gem install md2man
     ```
 - convert to perfect `.roff` file (`man` page)
     ```bash
-    md2man README.md > mcs.1
+    md2man-roff README.md > mcs.1
     ```
 - enjoy =)

cmapi/mcs_cluster_tool/__main__.py

@@ -36,7 +36,13 @@
 app.command(
     'dbrm_restore', rich_help_panel='Tools commands'
 )(restore_commands.dbrm_restore)
-app.command('cskeys', rich_help_panel='Tools commands')(tools_commands.cskeys)
+app.command(
+    'cskeys', rich_help_panel='Tools commands',
+    short_help=(
+        'Generates a random AES encryption key and init vector and writes '
+        'them to disk.'
+    )
+)(tools_commands.cskeys)
 app.command(
     'cspasswd', rich_help_panel='Tools commands'
 )(tools_commands.cspasswd)

cmapi/mcs_cluster_tool/decorators.py

@@ -25,11 +25,16 @@ def wrapper(*args, **kwargs):
         except typer.BadParameter as err:
             logger.error('Bad command line parameter.')
             raise err
+        except typer.Exit as err:  # if some command used typer.Exit
+            #TODO: think about universal protocol to return json data and
+            #      plain text results.
+            return_code = err.exit_code
         except Exception:
             logger.error(
                 'Undefined error while command execution',
                 exc_info=True
             )
             typer.echo('Unknown error, check the log file.', err=True)
+
         raise typer.Exit(return_code)
     return wrapper