From 606600d0896506fa489315ed814dd1f5e1bc8d17 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Thu, 30 Nov 2023 14:11:21 -0300 Subject: [PATCH 1/6] feat: set number_of_replicas to zero and disable refresh_interval during the index reset --- lib/esse/index/indices.rb | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/esse/index/indices.rb b/lib/esse/index/indices.rb index c769d29..041be98 100644 --- a/lib/esse/index/indices.rb +++ b/lib/esse/index/indices.rb @@ -26,10 +26,10 @@ module ClassMethods # # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/ # @see Esse::Transport#create_index - def create_index(suffix: nil, **options) + def create_index(suffix: nil, body: nil, **options) options = CREATE_INDEX_RESERVED_KEYWORDS.merge(options) name = build_real_index_name(suffix) - definition = [settings_hash, mappings_hash].reduce(&:merge) + definition = body || [settings_hash, mappings_hash].reduce(&:merge) if options.delete(:alias) && name != index_name definition[:aliases] = { index_name => {} } @@ -48,13 +48,23 @@ def create_index(suffix: nil, **options) # @return [Hash] the elasticsearch response # # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html - def reset_index(suffix: index_suffix, import: true, reindex: false, **options) + def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: false, **options) cluster.throw_error_when_readonly! existing = [] suffix ||= Esse.timestamp suffix = Esse.timestamp while index_exist?(suffix: suffix).tap { |exist| existing << suffix if exist } - create_index(**options, suffix: suffix, alias: false) + if optimize + definition = [settings_hash, mappings_hash].reduce(&:merge) + number_of_replicas = definition.dig(Esse::SETTING_ROOT_KEY, :index, :number_of_replicas) + refresh_interval = definition.dig(Esse::SETTING_ROOT_KEY, :index, :refresh_interval) + new_number_of_replicas = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:number_of_replicas] = 0 + new_refresh_interval = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:refresh_interval] = '-1' + create_index(**options, suffix: suffix, alias: false, body: definition) + else + create_index(**options, suffix: suffix, alias: false) + end + if index_exist? && aliases.none? cluster.api.delete_index(index: index_name) end @@ -63,8 +73,11 @@ def reset_index(suffix: index_suffix, import: true, reindex: false, **options) elsif reindex && (_from = indices_pointing_to_alias).any? # @TODO: Reindex using the reindex API end + if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval + update_settings(suffix: suffix) + end update_aliases(suffix: suffix) - existing.each { |_s| delete_index!(**options, suffix: suffix) } + existing.each { |s| delete_index!(**options, suffix: s) } true end From 8cafcabd0a6c882ec5075404c64cc29d38600831 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Thu, 20 Jun 2024 13:42:11 -0300 Subject: [PATCH 2/6] close index before update settings --- lib/esse/index/indices.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/esse/index/indices.rb b/lib/esse/index/indices.rb index 041be98..1e95996 100644 --- a/lib/esse/index/indices.rb +++ b/lib/esse/index/indices.rb @@ -74,7 +74,9 @@ def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: fal # @TODO: Reindex using the reindex API end if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval + close(suffix: suffix) update_settings(suffix: suffix) + open(suffix: suffix) end update_aliases(suffix: suffix) existing.each { |s| delete_index!(**options, suffix: s) } From 52365cb5e0fb52979cddec02598dd03b18d9dbfe Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Thu, 20 Jun 2024 13:43:19 -0300 Subject: [PATCH 3/6] add self to confusion with Kernel#open --- lib/esse/index/indices.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/esse/index/indices.rb b/lib/esse/index/indices.rb index 1e95996..87c5d54 100644 --- a/lib/esse/index/indices.rb +++ b/lib/esse/index/indices.rb @@ -76,7 +76,7 @@ def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: fal if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval close(suffix: suffix) update_settings(suffix: suffix) - open(suffix: suffix) + self.open(suffix: suffix) end update_aliases(suffix: suffix) existing.each { |s| delete_index!(**options, suffix: s) } From 6830327e14fbaa82bceeb547d97e675677992406 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Fri, 28 Jun 2024 18:01:54 -0300 Subject: [PATCH 4/6] fix: can not update number of replicas in a closed index --- lib/esse/index/indices.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/esse/index/indices.rb b/lib/esse/index/indices.rb index 87c5d54..041be98 100644 --- a/lib/esse/index/indices.rb +++ b/lib/esse/index/indices.rb @@ -74,9 +74,7 @@ def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: fal # @TODO: Reindex using the reindex API end if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval - close(suffix: suffix) update_settings(suffix: suffix) - self.open(suffix: suffix) end update_aliases(suffix: suffix) existing.each { |s| delete_index!(**options, suffix: s) } From 1bbed09a4b34da6163e5df07f1b03c1479a69322 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Fri, 28 Jun 2024 18:24:38 -0300 Subject: [PATCH 5/6] only delete existing index when the alias name is a real index --- lib/esse/index/indices.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/esse/index/indices.rb b/lib/esse/index/indices.rb index 041be98..e82abaf 100644 --- a/lib/esse/index/indices.rb +++ b/lib/esse/index/indices.rb @@ -50,9 +50,14 @@ def create_index(suffix: nil, body: nil, **options) # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: false, **options) cluster.throw_error_when_readonly! - existing = [] - suffix ||= Esse.timestamp - suffix = Esse.timestamp while index_exist?(suffix: suffix).tap { |exist| existing << suffix if exist } + + remove_root_in_use_index_named = nil + if index_name == index_name(suffix: suffix) && index_exist?(suffix: suffix) + remove_root_in_use_index_named = index_name + suffix = Esse.timestamp + elsif suffix.nil? || index_exist?(suffix: suffix) + suffix = Esse.timestamp + end if optimize definition = [settings_hash, mappings_hash].reduce(&:merge) @@ -73,11 +78,15 @@ def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: fal elsif reindex && (_from = indices_pointing_to_alias).any? # @TODO: Reindex using the reindex API end + if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval update_settings(suffix: suffix) end + + cluster.api.delete_index(index: remove_root_in_use_index_named) if remove_root_in_use_index_named + update_aliases(suffix: suffix) - existing.each { |s| delete_index!(**options, suffix: s) } + true end From ec277ed7625a110d93c366460b8bd56a0b6d08e4 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Mon, 1 Jul 2024 11:23:34 -0300 Subject: [PATCH 6/6] fix: fixing broken specs --- lib/esse/index/indices.rb | 11 ++--------- spec/esse/cluster_spec.rb | 12 +++++++----- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/lib/esse/index/indices.rb b/lib/esse/index/indices.rb index e82abaf..16e7c0f 100644 --- a/lib/esse/index/indices.rb +++ b/lib/esse/index/indices.rb @@ -51,13 +51,8 @@ def create_index(suffix: nil, body: nil, **options) def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: false, **options) cluster.throw_error_when_readonly! - remove_root_in_use_index_named = nil - if index_name == index_name(suffix: suffix) && index_exist?(suffix: suffix) - remove_root_in_use_index_named = index_name - suffix = Esse.timestamp - elsif suffix.nil? || index_exist?(suffix: suffix) - suffix = Esse.timestamp - end + suffix ||= Esse.timestamp + suffix = Esse.timestamp while index_exist?(suffix: suffix) if optimize definition = [settings_hash, mappings_hash].reduce(&:merge) @@ -83,8 +78,6 @@ def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: fal update_settings(suffix: suffix) end - cluster.api.delete_index(index: remove_root_in_use_index_named) if remove_root_in_use_index_named - update_aliases(suffix: suffix) true diff --git a/spec/esse/cluster_spec.rb b/spec/esse/cluster_spec.rb index 9324c54..53e7cc3 100644 --- a/spec/esse/cluster_spec.rb +++ b/spec/esse/cluster_spec.rb @@ -199,13 +199,15 @@ it 'retuns an instance of elasticsearch as default' do expect(model.instance_variable_get(:@client)).to eq(nil) + classes = [] + if defined? Elasticsearch::Client # Elasticsearch-ruby >= 8.0 + classes << Elasticsearch::Client + end if defined? Elasticsearch::Transport::Client - expect(model.client).to be_an_instance_of(Elasticsearch::Transport::Client) - expect(model.instance_variable_get(:@client)).to be_an_instance_of(Elasticsearch::Transport::Client) - else # Elasticsearch-ruby >= 8.0 - expect(model.client).to be_an_instance_of(Elasticsearch::Client) - expect(model.instance_variable_get(:@client)).to be_an_instance_of(Elasticsearch::Client) + classes << Elasticsearch::Transport::Client end + expect(classes).to include(model.client.class) + expect(model.client).to be(model.instance_variable_get(:@client)) end it 'store connection using default key' do