Add a base implementation for new MaxScale repository parser (refs #29314)

Author: amvasilyev

.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
   TargetRubyVersion: 2.6
+  NewCops: enable
 
 Layout/HeredocIndentation:
   Enabled: false
@@ -8,7 +9,7 @@ Metrics/AbcSize:
   Enabled: false
 
 Metrics/BlockLength:
-  ExcludedMethods:
+  IgnoredMethods:
     - context
     - describe
 
@@ -36,3 +37,5 @@ Style/MultilineBlockChain:
 Style/NegatedIf:
   Enabled: false
 
+Style/AccessorGrouping:
+  EnforcedStyle: separated

Gemfile.lock

@@ -4,12 +4,8 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
-      aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
-      aws-sigv4 (~> 1.1)
-      jmespath (~> 1.0)
-    aws-eventstream (1.1.1)
-    aws-partitions (1.448.0)
+    aws-eventstream (1.2.0)
+    aws-partitions (1.537.0)
     aws-sdk-core (3.114.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
@@ -18,7 +14,7 @@ GEM
     aws-sdk-ec2 (1.235.0)
       aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.3)
+    aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
@@ -80,24 +76,24 @@ GEM
       mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
     os (1.1.1)
-    parallel (1.20.1)
-    parser (3.0.1.0)
+    parallel (1.21.0)
+    parser (3.0.3.1)
       ast (~> 2.4.1)
     pastel (0.8.0)
       tty-color (~> 0.5)
-    pry (0.14.0)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.8.0)
       byebug (~> 11.0)
       pry (~> 0.10)
-    pry-doc (1.1.0)
+    pry-doc (1.2.0)
       pry (~> 0.11)
       yard (~> 0.9.11)
     public_suffix (4.0.6)
     racc (1.5.2)
     rainbow (3.0.0)
-    rake (13.0.3)
+    rake (13.0.6)
     regexp_parser (2.1.1)
     representable (3.0.4)
       declarative (< 0.1.0)
@@ -119,18 +115,18 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.12.1)
+    rspec-support (3.10.3)
+    rubocop (1.23.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.12.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.13.0)
+      parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.2)
     sem_version (2.0.1)
@@ -152,16 +148,18 @@ GEM
       strings (~> 0.2.0)
       tty-screen (~> 0.8)
     uber (0.1.0)
-    unicode-display_width (1.7.0)
+    unicode-display_width (1.8.0)
     unicode_utils (1.4.0)
     virtus (1.0.5)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
       equalizer (~> 0.0, >= 0.0.9)
+    webrick (1.7.0)
     workers (0.6.1)
     xdg (5.1.0)
-    yard (0.9.26)
+    yard (0.9.27)
+      webrick (~> 1.7.0)
 
 PLATFORMS
   ruby
@@ -190,4 +188,4 @@ DEPENDENCIES
   xdg
 
 BUNDLED WITH
-   2.2.15
+   2.2.22

config/generate_repository_config.yaml

@@ -1,11 +1,10 @@
 mariadb:
-  repo:
-    rpm:
-      path: http://yum.mariadb.org/
-      key: https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
-    deb:
-      path: http://mirror.netinch.com/pub/mariadb/repo/
-      key: "0xF1656F24C74CD1D8"
+  rpm:
+    path: https://dlm.mariadb.com/browse/mariadb_server/
+    key: https://downloads.mariadb.com/MariaDB/MariaDB-Server-GPG-KEY
+  deb:
+    path: https://dlm.mariadb.com/browse/mariadb_server/
+    key: ["0x8167EE24", "0xE3C94F49", "0xcbcb082a1bb943db", "0xf1656f24c74cd1d8", "0x135659e928c12247"]
 columnstore:
   repo:
     rpm:
@@ -20,19 +19,12 @@ maxscale_ci:
     old_key: https://mdbe-ci-repo.mariadb.net/MariaDBManager-GPG-KEY.public
     new_key: https://mdbe-ci-repo.mariadb.net/MDBE.public
 maxscale:
-  repo:
-    rpm:
-      path: https://downloads.mariadb.com/MaxScale/
-      key: https://downloads.mariadb.com/MariaDB/MariaDB-Enterprise-GPG-KEY
-      old_keys:
-        - key: https://downloads.mariadb.com/MaxScale/MariaDB-MaxScale-GPG-KEY
-          versions_upper_bound: ["2.4.12", "2.5.3", "2.3.21"]
-    deb:
-      path: https://downloads.mariadb.com/MaxScale/
-      key: https://downloads.mariadb.com/MariaDB/MariaDB-Enterprise-GPG-KEY
-      old_keys:
-        - key: "135659e928c12247"
-          versions_upper_bound: ["2.4.12", "2.5.3", "2.3.21"]
+  rpm:
+    path: https://dlm.mariadb.com/browse/mariadbmaxscale/
+    key: https://downloads.mariadb.com/MariaDB/MariaDB-Enterprise-GPG-KEY
+  deb:
+    path: https://dlm.mariadb.com/browse/mariadbmaxscale/
+    key: ["0x8167EE24", "0xE3C94F49", "0xcbcb082a1bb943db", "0xf1656f24c74cd1d8", "0x135659e928c12247"]
 mdbe:
   repo:
     rpm:

core/commands/generate_product_repositories_command.rb

@@ -14,6 +14,7 @@
 require_relative 'generate_repository_partials/mdbe_ci_parser'
 require_relative 'generate_repository_partials/maxscale_ci_parser'
 require_relative 'generate_repository_partials/maxscale_ci_docker_parser'
+require_relative 'generate_repository_partials/maria_db_community_parser'
 require_relative 'generate_repository_partials/maxscale_parser'
 require_relative 'generate_repository_partials/mdbe_parser'
 require_relative 'generate_repository_partials/mariadb_parser'
@@ -181,14 +182,14 @@ def setup_destination_directory
   end
 
   # Use data provided via constructor to configure the command.
-  def configure_command
+  def configure_command(threads_count)
     load_configuration_file
     return false unless determine_products_to_parse
     return false unless determine_product_version_to_parse
 
     determine_number_of_attempts
     setup_destination_directory
-    Workers.pool.resize(10)
+    Workers.pool.resize(threads_count)
     true
   end
 
@@ -206,7 +207,6 @@ def extract_release_fields(release)
     end
   end
 
-
   # Write all information about releases to the JSON documents
   def write_repository(product_dir, releases)
     platforms = releases.map { |release| release[:platform] }.uniq
@@ -287,11 +287,15 @@ def parse_repository(product, product_config)
     when 'maxscale_ci_docker'
       MaxscaleCiDockerParser.parse(@ui, @env.tool_config)
     when 'maxscale'
-      MaxscaleParser.parse(product_config, @product_version, @ui, @logger)
+      MariaDBCommunityParser.parse(product_config, MariaDBCommunityParser::MAXSCALE_SERVER,
+                                   'maxscale', @product_version,
+                                   @ui, @logger)
     when 'mdbe'
       MdbeParser.parse(product_config, @env.mdbe_private_key, 'MariaDB Enterprise Server', 'mdbe')
     when 'mariadb'
-      MariadbParser.parse(product_config, @product_version, @ui, @logger)
+      MariaDBCommunityParser.parse(product_config, MariaDBCommunityParser::MARIADB_COMMUNITY,
+                                   'mariadb', @product_version,
+                                   @ui, @logger)
     when 'columnstore'
       ColumnstoreParser.parse(product_config, @product_version, @ui, @logger)
     when 'mysql'
@@ -342,7 +346,7 @@ def execute
       show_help
       return SUCCESS_RESULT
     end
-    return ARGUMENT_ERROR_RESULT unless configure_command
+    return ARGUMENT_ERROR_RESULT unless configure_command(@env.threads_count)
 
     remainning_products = @products.dup
     @attempts.times do

core/commands/generate_repository_partials/clustrix_parser.rb

@@ -21,7 +21,7 @@ def self.clustrix_tar_link?(link)
   end
 
   def self.get_clustrix_links(path)
-    uri = path.gsub(%r{([^:])\/+}, '\1/')
+    uri = path.gsub(%r{([^:])/+}, '\1/')
     doc = Nokogiri::HTML(URI.open(uri))
     all_links = doc.css('a')
     all_links.select do |link|
@@ -67,7 +67,7 @@ def self.get_clustrix_release_versions(config, private_key, link_name)
       get_clustrix_tar_links(path)
     end.flatten.map do |link|
       path = generate_url_by_link(path_uri, link)
-      version = link.content.match(/^.*\/xpand-(.+)\.[^.]+\.tar\.bz2$/).captures[0].strip
+      version = link.content.match(%r{^.*/xpand-(.+)\.[^.]+\.tar\.bz2$}).captures[0].strip
       { repo: path, version: version }
     end
     releases_info.push(generate_latest_release(releases_info))

core/commands/generate_repository_partials/columnstore_parser.rb

@@ -8,8 +8,10 @@ module ColumnstoreParser
 
   def self.parse(config, product_version, log, logger)
     releases = []
-    releases.concat(parse_columnstore_rpm_repository(config['repo']['rpm'], product_version, log, logger))
-    releases.concat(parse_columnstore_deb_repository(config['repo']['deb'], product_version, log, logger))
+    releases.concat(parse_columnstore_rpm_repository(config['repo']['rpm'], product_version, log,
+                                                     logger))
+    releases.concat(parse_columnstore_deb_repository(config['repo']['deb'], product_version, log,
+                                                     logger))
     releases
   end
 
@@ -39,7 +41,7 @@ def self.parse_columnstore_deb_repository(config, product_version, log, logger)
       log, logger,
       save_as_field(:version),
       append_url(%w[repo]),
-      extract_field(:platform, %r{^(\p{Alpha}+)\p{Digit}+\/?$}, true),
+      extract_field(:platform, %r{^(\p{Alpha}+)\p{Digit}+/?$}, true),
       append_url(%w[dists]),
       save_as_field(:platform_version),
       lambda do |release, _|

core/commands/generate_repository_partials/galera_ci_parser.rb

@@ -11,46 +11,48 @@ def self.parse(config, product_version, mdbe_ci_config, galera_version, log, log
 
     auth_mdbe_ci_repo = mdbe_ci_config['mdbe_ci_repo']
     releases = []
-    releases.concat(parse_galera_ci_rpm_repository(config['repo'], product_version, auth_mdbe_ci_repo, galera_version, log, logger))
-    releases.concat(parse_galera_ci_deb_repository(config['repo'], product_version, auth_mdbe_ci_repo, galera_version, log, logger))
+    releases.concat(parse_galera_ci_rpm_repository(config['repo'], product_version,
+                                                   auth_mdbe_ci_repo, galera_version, log, logger))
+    releases.concat(parse_galera_ci_deb_repository(config['repo'], product_version,
+                                                   auth_mdbe_ci_repo, galera_version, log, logger))
     releases
   end
 
   def self.parse_galera_ci_rpm_repository(config, product_version, auth, galera_version, log, logger)
     parse_repository(
-        config['path'], auth, add_auth_to_url(config['key'], auth), galera_version, product_version,
-        %w[galera],
-        ->(url, _) { url },
-        ->(package, _) { /#{package}/ },
-        log, logger,
-        save_as_field(:version),
-        split_rpm_platforms,
-        extract_field(:platform_version, %r{^(\p{Digit}+)\/?$}),
-        append_url(%w[x86_64 aarch64], :architecture),
-        lambda do |release, _|
-          release[:repo] = add_auth_to_url(release[:url], auth)
-          release
-        end
+      config['path'], auth, add_auth_to_url(config['key'], auth), galera_version, product_version,
+      %w[galera],
+      ->(url, _) { url },
+      ->(package, _) { /#{package}/ },
+      log, logger,
+      save_as_field(:version),
+      split_rpm_platforms,
+      extract_field(:platform_version, %r{^(\p{Digit}+)/?$}),
+      append_url(%w[x86_64 aarch64], :architecture),
+      lambda do |release, _|
+        release[:repo] = add_auth_to_url(release[:url], auth)
+        release
+      end
     )
   end
 
   def self.parse_galera_ci_deb_repository(config, product_version, auth, galera_version, log, logger)
     parse_repository(
-        config['path'], auth, add_auth_to_url(config['key'], auth), galera_version, product_version,
-        %w[galera],
-        ->(url, _) { generate_galera_ci_deb_full_url(url, logger, auth) },
-        ->(package, platform) { /#{package}.*#{platform}/ },
-        log, logger,
-        save_as_field(:version),
-        append_url(%w[apt], nil, true),
-        append_url(%w[dists]),
-        extract_deb_platforms,
-        set_deb_architecture(auth),
-        lambda do |release, _|
-          repo_path = add_auth_to_url(release[:repo_url], auth)
-          release[:repo] = "#{repo_path} #{release[:platform_version]} main"
-          release
-        end
+      config['path'], auth, add_auth_to_url(config['key'], auth), galera_version, product_version,
+      %w[galera],
+      ->(url, _) { generate_galera_ci_deb_full_url(url, logger, auth) },
+      ->(package, platform) { /#{package}.*#{platform}/ },
+      log, logger,
+      save_as_field(:version),
+      append_url(%w[apt], nil, true),
+      append_url(%w[dists]),
+      extract_deb_platforms,
+      set_deb_architecture(auth),
+      lambda do |release, _|
+        repo_path = add_auth_to_url(release[:repo_url], auth)
+        release[:repo] = "#{repo_path} #{release[:platform_version]} main"
+        release
+      end
     )
   end
 
@@ -62,7 +64,7 @@ def self.generate_galera_ci_deb_full_url(incorrect_url, logger, auth)
   end
 
   def self.generate_pool_link(url, logger, auth)
-    dir = get_directory_links("#{url}", logger, auth)[0].attributes['href'].value
+    dir = get_directory_links(url.to_s, logger, auth)[0].attributes['href'].value
     "#{url}/#{dir}"
   rescue OpenURI::HTTPError => e
     url