Allow to use mdbci generate repositories command to update a single release for a product (refs #23932)

Author: amvasilyev

console.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 require 'pry'
-current_path = File.expand_path('../core', __FILE__)
+current_path = File.expand_path('./core', __dir__)
 source_files = Dir.glob("#{current_path}/**/*.rb")
 source_files.each do |file|
   require file
 end
 # rubocop:disable Lint/Debugger
 binding.pry
+puts '' # Needed to make pry work
 # rubocop:enable Lint/Debugger

core/commands/generate_product_repositories_command.rb

@@ -4,7 +4,6 @@
 require 'optparse'
 require 'nokogiri'
 require 'pp'
-require 'tmpdir'
 require 'json'
 require 'fileutils'
 require 'logger'
@@ -13,6 +12,7 @@
 require_relative 'base_command'
 
 # The command generates the repository configuration
+# rubocop:disable Metrics/ClassLength
 class GenerateProductRepositoriesCommand < BaseCommand
   CONFIGURATION_FILE = 'generate_repository_config.yaml'
   PRODUCTS_DIR_NAMES = {
@@ -48,27 +48,31 @@ def show_help
 
 Supported options:
 
---maxscale-ci name of the repository on the MaxScale CI server to add to MaxScale CI product configuration. Required if generating configuration for MaxScale CI server.
 --configuration-file path to the configuration file to use during generation. Optional.
 --product name of the product to generate repository configuration for. Optional.
+--product-version version of the product to generate configuration for.
 --attempts number of attempts to try to get data from the remote repository. Default is 3 attempts.
 
 In order to generate repo.d for all products using the default configuration.
 
-  mdbci #{COMMAND_NAME} --maxscale-ci develop
+  mdbci #{COMMAND_NAME}
 
 You can create custom configuration file and use it during the repository creation:
 
-  mdbci #{COMMAND_NAME} --maxscale-ci develop --configuration-file ~/mdbci/config/generate_repository_config.yaml
+  mdbci #{COMMAND_NAME} --configuration-file ~/mdbci/config/generate_repository_config.yaml
 
 In order to specify the target directory pass it as the first parameter to the script.
 
-  mdbci #{COMMAND_NAME} --maxscale-ci develop ~/mdbci/repo.d
+  mdbci #{COMMAND_NAME} ~/mdbci/repo.d
 
 In orded to generate configuration for a specific product use --product option.
 
   mdbci #{COMMAND_NAME} --product mdbe
 
+In order to generate configuration for a specific product version use --product-version option. You must also specify the name of the product to generate configuration for.
+
+  mdbci #{COMMAND_NAME} --product maxscale-ci --product-version develop
+
 In order to specify the number of retries for repository configuration use --attempts option.
 
   mdbci generate-product-repositories --product columnstore --attempts 5
@@ -115,8 +119,8 @@ def load_configuration_file
   def determine_products_to_parse
     if @env.nodeProduct
       unless PRODUCTS_DIR_NAMES.key?(@env.nodeProduct)
-        show_error_and_help("Unknown product #{@env.nodeProduct}.\n"\
-                            "Known products: #{PRODUCTS_DIR_NAMES.keys.join(', ')}")
+        error_and_log("Unknown product #{@env.nodeProduct}.\n"\
+                      "Known products: #{PRODUCTS_DIR_NAMES.keys.join(', ')}")
         return false
       end
       @products = [@env.nodeProduct]
@@ -127,6 +131,17 @@ def determine_products_to_parse
     true
   end
 
+  def determine_product_version_to_parse
+    if @env.productVersion
+      unless @env.nodeProduct
+        error_and_log('When specifying product version you must also specify the product.')
+        return false
+      end
+      @product_version = @env.productVersion
+    end
+    true
+  end
+
   def determine_number_of_attempts
     @attempts = if @env.attempts
                   @env.attempts.to_i
@@ -147,11 +162,15 @@ def setup_destination_directory
 
   # Use data provided via constructor to configure the command.
   def configure_command
+    if @env.show_help
+      show_help
+      return false
+    end
     load_configuration_file
     return false unless determine_products_to_parse
+    return false unless determine_product_version_to_parse
     determine_number_of_attempts
     setup_destination_directory
-    @maxscale_ci = @env.maxscale_ci
     Workers.pool.resize(10)
     true
   end
@@ -182,7 +201,7 @@ def parse_maxscale_ci(config)
     releases = []
     releases.concat(parse_maxscale_ci_rpm_repository(config['repo']['rpm']))
     releases.concat(parse_maxscale_ci_deb_repository(config['repo']['deb']))
-    write_repository(releases)
+    releases
   end
 
   def parse_maxscale_ci_rpm_repository(config)
@@ -219,7 +238,7 @@ def parse_maxscale(config)
     releases = []
     releases.concat(parse_maxscale_rpm_repository(config['repo']['rpm']))
     releases.concat(parse_maxscale_deb_repository(config['repo']['deb']))
-    write_repository(releases)
+    releases
   end
 
   def parse_maxscale_rpm_repository(config)
@@ -254,7 +273,7 @@ def parse_mdbe(config)
     releases = []
     releases.concat(parse_mdbe_rpm_repository(config['repo']['rpm']))
     releases.concat(parse_mdbe_deb_repository(config['repo']['deb']))
-    write_repository(releases)
+    releases
   end
 
   def parse_mdbe_rpm_repository(config)
@@ -292,15 +311,15 @@ def parse_galera(config)
     version_regexp = %r{^(\p{Digit}+\.\p{Digit}+)\-galera\/?$}
     releases.concat(parse_mariadb_rpm_repository(config['repo']['rpm'], 'galera', version_regexp))
     releases.concat(parse_mariadb_deb_repository(config['repo']['deb'], 'galera', version_regexp))
-    write_repository(releases)
+    releases
   end
 
   def parse_mariadb(config)
     releases = []
     version_regexp = %r{^(\p{Digit}+\.\p{Digit}+(\.\p{Digit}+)?)\/?$}
     releases.concat(parse_mariadb_rpm_repository(config['repo']['rpm'], 'mariadb', version_regexp))
     releases.concat(parse_mariadb_deb_repository(config['repo']['deb'], 'mariadb', version_regexp))
-    write_repository(releases)
+    releases
   end
 
   def parse_mariadb_rpm_repository(config, product, version_regexp)
@@ -335,7 +354,7 @@ def parse_columnstore(config)
     releases = []
     releases.concat(parse_columnstore_rpm_repository(config['repo']['rpm']))
     releases.concat(parse_columnstore_deb_repository(config['repo']['deb']))
-    write_repository(releases)
+    releases
   end
 
   def parse_columnstore_rpm_repository(config)
@@ -372,7 +391,7 @@ def parse_mysql(config)
     releases = []
     releases.concat(parse_mysql_rpm_repository(config['repo']['rpm']))
     releases.concat(parse_mysql_deb_repository(config['repo']['deb']))
-    write_repository(releases)
+    releases
   end
 
   def parse_mysql_deb_repository(config)
@@ -405,23 +424,6 @@ def parse_mysql_rpm_repository(config)
     )
   end
 
-  # Write all information about releases to the JSON documents
-  def write_repository(releases)
-    platforms = releases.map { |release| release[:platform] }.uniq
-    platforms.each do |platform|
-      FileUtils.mkdir_p("#{@directory}/#{platform}")
-      releases_by_version = Hash.new { |hash, key| hash[key] = [] }
-      releases.each do |release|
-        next if release[:platform] != platform
-        releases_by_version[release[:version]] << extract_release_fields(release)
-      end
-      releases_by_version.each_pair do |version, version_releases|
-        File.write("#{@directory}/#{platform}/#{version}.json",
-                   JSON.pretty_generate(version_releases))
-      end
-    end
-  end
-
   STORED_KEYS = %i[repo repo_key platform platform_version product version].freeze
   # Extract only required fields from the passed release before writing it to the file
   def extract_release_fields(release)
@@ -431,6 +433,18 @@ def extract_release_fields(release)
     end
   end
 
+  # Method filters releases, removing empty ones and by version, if it required
+  def filter_releases(releases)
+    next_releases = releases.reject(&:nil?)
+    next_releases.select do |release|
+      if @product_version.nil? || release[:version].nil?
+        true
+      else
+        release[:version] == @product_version
+      end
+    end
+  end
+
   # Parse the repository and provide required configurations
   def parse_repository(base_url, key, product, *steps)
     # Recursively go through the site and apply steps on each level
@@ -444,10 +458,17 @@ def parse_repository(base_url, key, product, *steps)
         end
         apply_step_to_links(step, links, release)
       end
-      next_releases.flatten.reject(&:nil?)
+      filter_releases(next_releases.flatten)
     end
-    # Add repository key and product to all releases
-    result.each do |release|
+    add_key_and_product_to_releases(result, key, product)
+  end
+
+  # Append key and product to the releases
+  # @param releases [Array<Hash>] list of releases
+  # @param key [String] text to put into key field
+  # @param product [String] name of the product
+  def add_key_and_product_to_releases(releases, key, product)
+    releases.each do |release|
       release[:repo_key] = key
       release[:product] = product
     end
@@ -554,22 +575,53 @@ def append_url(paths, key = nil, save_path = false)
     end
   end
 
-  # Create repository
+  # Write all information about releases to the JSON documents
+  def write_repository(product_dir, releases)
+    platforms = releases.map { |release| release[:platform] }.uniq
+    platforms.each do |platform|
+      configuration_directory = File.join(product_dir, platform)
+      FileUtils.mkdir_p(configuration_directory)
+      releases_by_version = Hash.new { |hash, key| hash[key] = [] }
+      releases.each do |release|
+        next if release[:platform] != platform
+        releases_by_version[release[:version]] << extract_release_fields(release)
+      end
+      releases_by_version.each_pair do |version, version_releases|
+        File.write(File.join(configuration_directory, "#{version}.json"),
+                   JSON.pretty_generate(version_releases))
+      end
+    end
+  end
+
+  # Create repository in the target directory.
+  def write_product(product, releases)
+    info_and_log("Writing generated configuration for #{product} to the repository.")
+    product_name = PRODUCTS_DIR_NAMES[product]
+    product_dir = File.join(@destination, product_name)
+    if @product_version.nil?
+      FileUtils.rm_rf(product_dir, secure: true)
+      FileUtils.mkdir_p(product_dir)
+    else
+      releases = releases.select do |release|
+        release[:version] == @product_version
+      end
+    end
+    write_repository(product_dir, releases)
+  end
+
+  # Create repository by calling appropriate method using reflection and writing results
+  # @param product [String] name of the product to generate
+  # @returns [Boolean] whether generation was succesfull or not
   def create_repository(product)
     info_and_log("Generating repository configuration for #{product}")
-    FileUtils.rm_rf("#{@directory}/.", secure: true)
     begin
-      send("parse_#{product}".to_sym, @config[product])
+      releases = send("parse_#{product}".to_sym, @config[product])
     rescue StandardError => error
       error_and_log("#{product} was not generated. Try again.")
       error_and_log_error(error)
       return false
     end
-    info_and_log("Copying generated configuration for #{product} to the repository.")
-    product_name = PRODUCTS_DIR_NAMES[product]
-    FileUtils.mkdir_p("#{@destination}/#{product_name}")
-    FileUtils.rm_rf("#{@destination}/#{product_name}", secure: true)
-    FileUtils.cp_r("#{@directory}/.", "#{@destination}/#{product_name}")
+    write_product(product, releases)
     true
   end
 
@@ -586,8 +638,6 @@ def print_summary(products_with_errors)
   # Starting point of the application
   def execute
     return ARGUMENT_ERROR_RESULT unless configure_command
-    @directory = Dir.mktmpdir(COMMAND_NAME)
-    info_and_log("Writing intermediate results to #{@directory}")
     remainning_products = @products.dup
     @attempts.times do
       remainning_products = remainning_products.reject do |product|
@@ -596,7 +646,7 @@ def execute
       break if remainning_products.empty?
     end
     print_summary(remainning_products)
-    FileUtils.rm_rf(@directory)
     SUCCESS_RESULT
   end
 end
+# rubocop:enable Metrics/ClassLength

core/session.rb

@@ -46,7 +46,6 @@ class Session
   attr_accessor :boxes_dir
   attr_accessor :mdbciDir
   attr_accessor :mdbci_dir
-  attr_accessor :maxscale_ci
   attr_accessor :starting_dir
   attr_accessor :working_dir
   attr_accessor :nodeProduct
@@ -59,6 +58,7 @@ class Session
   attr_accessor :node_name
   attr_accessor :snapshot_name
   attr_accessor :ipv6
+  attr_accessor :show_help
 
   PLATFORM = 'platform'
   VAGRANT_NO_PARALLEL = '--no-parallel'

mdbci

@@ -16,10 +16,10 @@ opts = GetoptLong.new(
   ['--command', '-c', GetoptLong::REQUIRED_ARGUMENT],
   ['--configuration-file', GetoptLong::REQUIRED_ARGUMENT],
   ['--field', '-f', GetoptLong::REQUIRED_ARGUMENT],
+  ['--help', GetoptLong::NO_ARGUMENT],
   ['--ipv6', GetoptLong::NO_ARGUMENT],
   ['--key', '-k', GetoptLong::REQUIRED_ARGUMENT],
   ['--keep-template', GetoptLong::NO_ARGUMENT],
-  ['--maxscale-ci', GetoptLong::REQUIRED_ARGUMENT],
   ['--node-name', GetoptLong::REQUIRED_ARGUMENT],
   ['--override', '-w', GetoptLong::NO_ARGUMENT],
   ['--path-to-nodes', GetoptLong::REQUIRED_ARGUMENT],
@@ -117,8 +117,8 @@ begin
     when '--configuration-file'
       session.configuration_file = File.absolute_path(arg)
 
-    when '--maxscale-ci'
-      session.maxscale_ci = arg
+    when '--help'
+      session.show_help = true
 
     else
       out.error 'Option is not supported: ' + opt