diff --git a/CHANGELOG.md b/CHANGELOG.md index b7ef001..77bdcbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.3.2 - 2024-07-12 +* fix bulk indexing routing issue +* add `attributes:` to the `Esse::Repository.each_serialized_batch` to preload `lazy_document_attributes` +* Stop stringifying the `lazy_document_attributes` attribute name +* The `Esse::Repository.update_documents_attribute` was not working when calling with a single hash as document + ## 0.3.0 - 2024-07-10 * Extend bulk indexing API to support `update`. * Last attempt of bulk, index each document individually if the bulk fails. diff --git a/Gemfile.lock b/Gemfile.lock index eea03be..08ebc61 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.elasticsearch-1.x.lock b/gemfiles/Gemfile.elasticsearch-1.x.lock index 4f87c38..f40f9d0 100644 --- a/gemfiles/Gemfile.elasticsearch-1.x.lock +++ b/gemfiles/Gemfile.elasticsearch-1.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.elasticsearch-2.x.lock b/gemfiles/Gemfile.elasticsearch-2.x.lock index f044592..55f6464 100644 --- a/gemfiles/Gemfile.elasticsearch-2.x.lock +++ b/gemfiles/Gemfile.elasticsearch-2.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.elasticsearch-5.x.lock b/gemfiles/Gemfile.elasticsearch-5.x.lock index 8f29e40..b86d916 100644 --- a/gemfiles/Gemfile.elasticsearch-5.x.lock +++ b/gemfiles/Gemfile.elasticsearch-5.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.elasticsearch-6.x.lock b/gemfiles/Gemfile.elasticsearch-6.x.lock index 5646c71..f72a3e4 100644 --- a/gemfiles/Gemfile.elasticsearch-6.x.lock +++ b/gemfiles/Gemfile.elasticsearch-6.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.elasticsearch-7.x.lock b/gemfiles/Gemfile.elasticsearch-7.x.lock index 20dda77..da2ce55 100644 --- a/gemfiles/Gemfile.elasticsearch-7.x.lock +++ b/gemfiles/Gemfile.elasticsearch-7.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.elasticsearch-8.x.lock b/gemfiles/Gemfile.elasticsearch-8.x.lock index d93d151..09a5d11 100644 --- a/gemfiles/Gemfile.elasticsearch-8.x.lock +++ b/gemfiles/Gemfile.elasticsearch-8.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.opensearch-1.x.lock b/gemfiles/Gemfile.opensearch-1.x.lock index 37cca80..3e12756 100644 --- a/gemfiles/Gemfile.opensearch-1.x.lock +++ b/gemfiles/Gemfile.opensearch-1.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/gemfiles/Gemfile.opensearch-2.x.lock b/gemfiles/Gemfile.opensearch-2.x.lock index 4b62c59..aa976e8 100644 --- a/gemfiles/Gemfile.opensearch-2.x.lock +++ b/gemfiles/Gemfile.opensearch-2.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.3.1) + esse (0.3.2) multi_json thor (>= 0.19) diff --git a/lib/esse/document.rb b/lib/esse/document.rb index 7c5cd0c..97bbb4a 100644 --- a/lib/esse/document.rb +++ b/lib/esse/document.rb @@ -93,7 +93,7 @@ def ==(other) def doc_header { _id: id }.tap do |h| h[:_type] = type if type - h[:_routing] = routing if routing? + h[:routing] = routing if routing? end end @@ -102,6 +102,7 @@ def inspect value = send(attr) "#{attr}: #{value.inspect}" if value end.compact.join(', ') + attributes << " mutations: #{@__mutations__.inspect}" if @__mutations__ "#<#{self.class.name || 'Esse::Document'} #{attributes}>" end diff --git a/lib/esse/index/documents.rb b/lib/esse/index/documents.rb index f7a996d..fe61f7f 100644 --- a/lib/esse/index/documents.rb +++ b/lib/esse/index/documents.rb @@ -209,7 +209,9 @@ def import(*repo_types, context: {}, eager_include_document_attributes: false, l doc_attrs[:lazy] = repo.lazy_document_attribute_names(lazy_update_document_attributes) doc_attrs[:lazy] -= doc_attrs[:eager] - repo.each_serialized_batch(**(context || {})) do |batch| + context ||= {} + context[:lazy_attributes] = doc_attrs[:eager] if doc_attrs[:eager].any? + repo.each_serialized_batch(**context) do |batch| # Elasticsearch 6.x and older have multiple types per index. # This gem supports multiple types per index for backward compatibility, but we recommend to update # your elasticsearch to a at least 7.x version and use a single type per index. @@ -219,17 +221,10 @@ def import(*repo_types, context: {}, eager_include_document_attributes: false, l kwargs = { suffix: suffix, type: repo_name, **options } cluster.may_update_type!(kwargs) - doc_attrs[:eager].each do |attr_name| - repo.retrieve_lazy_attribute_values(attr_name, *batch.reject(&:ignore_on_index?)).each do |doc_header, value| - doc = batch.find { |d| doc_header.id == d.id && doc_header.type == d.type && doc_header.routing == d.routing } - doc&.mutate(attr_name) { value } - end - end - bulk(**kwargs, index: batch) doc_attrs[:lazy].each do |attr_name| - partial_docs = repo.documents_for_lazy_attribute(attr_name, *batch.reject(&:ignore_on_index?)) + partial_docs = repo.documents_for_lazy_attribute(attr_name, batch.reject(&:ignore_on_index?)) next if partial_docs.empty? bulk(**kwargs, update: partial_docs) diff --git a/lib/esse/lazy_document_header.rb b/lib/esse/lazy_document_header.rb index bc3f1b6..c3039bd 100644 --- a/lib/esse/lazy_document_header.rb +++ b/lib/esse/lazy_document_header.rb @@ -4,7 +4,7 @@ module Esse class LazyDocumentHeader def self.coerce_each(values) arr = [] - Array(values).map do |value| + Esse::ArrayUtils.wrap(values).map do |value| instance = coerce(value) arr << instance if instance&.valid? end @@ -24,7 +24,7 @@ def self.coerce(value) when :_id, :id, '_id', 'id' :_id when :_routing, :routing, '_routing', 'routing' - :_routing + :routing when :_type, :type, '_type', 'type' :_type else @@ -58,7 +58,7 @@ def type end def routing - @attributes[:_routing] + @attributes[:routing] end def to_doc(source = {}) diff --git a/lib/esse/primitives.rb b/lib/esse/primitives.rb index 56fb9a4..6d035cb 100644 --- a/lib/esse/primitives.rb +++ b/lib/esse/primitives.rb @@ -2,3 +2,4 @@ require_relative 'primitives/hstring' require_relative 'primitives/hash_utils' +require_relative 'primitives/array_utils' diff --git a/lib/esse/primitives/array_utils.rb b/lib/esse/primitives/array_utils.rb new file mode 100644 index 0000000..b6f9f41 --- /dev/null +++ b/lib/esse/primitives/array_utils.rb @@ -0,0 +1,17 @@ +module Esse + # The idea here is to add useful methods to the ruby standard objects without + # monkey patching them + module ArrayUtils + module_function + + def wrap(object) + if object.nil? + [] + elsif object.respond_to?(:to_ary) + object.to_ary || [object] + else + [object] + end + end + end +end diff --git a/lib/esse/repository/documents.rb b/lib/esse/repository/documents.rb index 02cf8af..0d2a7d6 100644 --- a/lib/esse/repository/documents.rb +++ b/lib/esse/repository/documents.rb @@ -7,20 +7,20 @@ def import(**kwargs) index.import(repo_name, **kwargs) end - def update_documents_attribute(name, *ids_or_doc_headers, **kwargs) - batch = documents_for_lazy_attribute(name, *ids_or_doc_headers) + def update_documents_attribute(name, ids_or_doc_headers = [], kwargs = {}) + batch = documents_for_lazy_attribute(name, ids_or_doc_headers) return if batch.empty? - index.bulk(**kwargs, update: batch) + index.bulk(**kwargs.transform_keys(&:to_sym), update: batch) end - def documents_for_lazy_attribute(name, *ids_or_doc_headers) - retrieve_lazy_attribute_values(name, *ids_or_doc_headers).map do |doc_header, datum| + def documents_for_lazy_attribute(name, ids_or_doc_headers) + retrieve_lazy_attribute_values(name, ids_or_doc_headers).map do |doc_header, datum| doc_header.to_doc(name => datum) end end - def retrieve_lazy_attribute_values(name, *ids_or_doc_headers) + def retrieve_lazy_attribute_values(name, ids_or_doc_headers) unless lazy_document_attribute?(name) raise ArgumentError, <<~MSG The attribute `#{name}` is not defined as a lazy document attribute. diff --git a/lib/esse/repository/lazy_document_attributes.rb b/lib/esse/repository/lazy_document_attributes.rb index 527cf77..cfb40be 100644 --- a/lib/esse/repository/lazy_document_attributes.rb +++ b/lib/esse/repository/lazy_document_attributes.rb @@ -15,23 +15,23 @@ def lazy_document_attribute_names(all = true) when true lazy_document_attributes.keys else - Array(all).map(&:to_s) & lazy_document_attributes.keys + filtered = Array(all).map(&:to_s) + lazy_document_attributes.keys.select { |name| filtered.include?(name.to_s) } end end - def lazy_document_attribute?(attr_name) - lazy_document_attributes.key?(attr_name.to_s) - end - def fetch_lazy_document_attribute(attr_name) - klass, kwargs = lazy_document_attributes.fetch(attr_name.to_s) + klass, kwargs = lazy_document_attributes.fetch(attr_name) klass.new(**kwargs) rescue KeyError raise ArgumentError, format('Attribute %p is not defined as a lazy document attribute', attr: attr_name) end def lazy_document_attribute(attr_name, klass = nil, **kwargs, &block) - if lazy_document_attribute?(attr_name) + if attr_name.nil? + raise ArgumentError, 'Attribute name is required to define a lazy document attribute' + end + if lazy_document_attribute?(attr_name.to_sym) || lazy_document_attribute?(attr_name.to_s) raise ArgumentError, format('Attribute %p is already defined as a lazy document attribute', attr: attr_name) end @@ -40,11 +40,11 @@ def lazy_document_attribute(attr_name, klass = nil, **kwargs, &block) klass = Class.new(Esse::DocumentLazyAttribute) do define_method(:call, &block) end - @lazy_document_attributes[attr_name.to_s] = [klass, kwargs] + @lazy_document_attributes[attr_name] = [klass, kwargs] elsif klass.is_a?(Class) && klass <= Esse::DocumentLazyAttribute - @lazy_document_attributes[attr_name.to_s] = [klass, kwargs] + @lazy_document_attributes[attr_name] = [klass, kwargs] elsif klass.is_a?(Class) && klass.instance_methods.include?(:call) - @lazy_document_attributes[attr_name.to_s] = [klass, kwargs] + @lazy_document_attributes[attr_name] = [klass, kwargs] elsif klass.nil? raise ArgumentError, format('A block or a class that responds to `call` is required to define a lazy document attribute') else @@ -53,6 +53,12 @@ def lazy_document_attribute(attr_name, klass = nil, **kwargs, &block) ensure @lazy_document_attributes&.freeze end + + protected + + def lazy_document_attribute?(attr_name) + lazy_document_attributes.key?(attr_name) + end end extend ClassMethods diff --git a/lib/esse/repository/object_document_mapper.rb b/lib/esse/repository/object_document_mapper.rb index 989e92c..452a517 100644 --- a/lib/esse/repository/object_document_mapper.rb +++ b/lib/esse/repository/object_document_mapper.rb @@ -74,11 +74,21 @@ def collection(collection_klass = nil, **_, &block) # @param [Hash] kwargs The context # @return [Enumerator] The enumerator # @yield [Array, **context] serialized collection and the optional context from the collection - def each_serialized_batch(**kwargs) + def each_serialized_batch(lazy_attributes: false, **kwargs) each_batch(**kwargs) do |*args| batch, collection_context = args collection_context ||= {} entries = [*batch].map { |entry| serialize(entry, **collection_context) }.compact + if lazy_attributes + attrs = lazy_attributes.is_a?(Array) ? lazy_attributes : lazy_document_attribute_names(lazy_attributes) + attrs.each do |attr_name| + retrieve_lazy_attribute_values(attr_name, entries).each do |doc_header, value| + doc = entries.find { |d| doc_header.id.to_s == d.id.to_s && doc_header.type == d.type && doc_header.routing == d.routing } + doc&.mutate(attr_name) { value } + end + end + end + yield entries, **kwargs end end diff --git a/lib/esse/version.rb b/lib/esse/version.rb index 8dd2e5a..bf88935 100644 --- a/lib/esse/version.rb +++ b/lib/esse/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Esse - VERSION = '0.3.1' + VERSION = '0.3.2' end diff --git a/spec/esse/document_spec.rb b/spec/esse/document_spec.rb index a31174b..82a99ee 100644 --- a/spec/esse/document_spec.rb +++ b/spec/esse/document_spec.rb @@ -91,19 +91,19 @@ def source context 'with data: true' do subject { document.to_bulk(data: true) } - it { is_expected.to eq(_id: 1, _type: 'foo', _routing: 'bar', timeout: 10, data: { foo: 'bar' }) } + it { is_expected.to eq(_id: 1, _type: 'foo', routing: 'bar', timeout: 10, data: { foo: 'bar' }) } end context 'with data: false' do subject { document.to_bulk(data: false) } - it { is_expected.to eq(_id: 1, _type: 'foo', _routing: 'bar', timeout: 10) } + it { is_expected.to eq(_id: 1, _type: 'foo', routing: 'bar', timeout: 10) } end context 'with operation: :update' do subject { document.to_bulk(data: true, operation: :update) } - it { is_expected.to eq(_id: 1, _type: 'foo', _routing: 'bar', timeout: 10, data: { doc: { foo: 'bar' } }) } + it { is_expected.to eq(_id: 1, _type: 'foo', routing: 'bar', timeout: 10, data: { doc: { foo: 'bar' } }) } end context 'when document does not have a routing' do @@ -116,21 +116,21 @@ def source context 'when document does not have a type' do it 'should not include the type' do allow(document).to receive(:type).and_return(nil) - expect(document.to_bulk(data: true)).to eq(_id: 1, _routing: 'bar', timeout: 10, data: { foo: 'bar' }) + expect(document.to_bulk(data: true)).to eq(_id: 1, routing: 'bar', timeout: 10, data: { foo: 'bar' }) end end context 'when document does not have a meta' do it 'should not include the meta' do allow(document).to receive(:meta).and_return({}) - expect(document.to_bulk(data: true)).to eq(_id: 1, _type: 'foo', _routing: 'bar', data: { foo: 'bar' }) + expect(document.to_bulk(data: true)).to eq(_id: 1, _type: 'foo', routing: 'bar', data: { foo: 'bar' }) end end context 'when document does not have a source' do it 'should not include the source' do allow(document).to receive(:source).and_return({}) - expect(document.to_bulk(data: true)).to eq(_id: 1, _type: 'foo', _routing: 'bar', timeout: 10, data: {}) + expect(document.to_bulk(data: true)).to eq(_id: 1, _type: 'foo', routing: 'bar', timeout: 10, data: {}) end end end @@ -156,7 +156,7 @@ def routing subject { document.doc_header } - it { is_expected.to eq(_id: 1, _type: 'foo', _routing: 'bar') } + it { is_expected.to eq(_id: 1, _type: 'foo', routing: 'bar') } context 'when document does not have a routing' do it 'should not include the routing' do @@ -168,7 +168,7 @@ def routing context 'when document does not have a type' do it 'should not include the type' do allow(document).to receive(:type).and_return(nil) - expect(document.doc_header).to eq(_id: 1, _routing: 'bar') + expect(document.doc_header).to eq(_id: 1, routing: 'bar') end end end diff --git a/spec/esse/integrations/elasticsearch-6/repository/documents_update_documents_attribute_spec.rb b/spec/esse/integrations/elasticsearch-6/repository/documents_update_documents_attribute_spec.rb new file mode 100644 index 0000000..2a78b3e --- /dev/null +++ b/spec/esse/integrations/elasticsearch-6/repository/documents_update_documents_attribute_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'support/shared_examples/repository_documents_update_documents_attribute' + +stack_describe 'elasticsearch', '6.x', Esse::Repository, '.update_documents_attribute' do + include_examples 'repository.update_documents_attribute' +end diff --git a/spec/esse/integrations/elasticsearch-7/repository/documents_update_documents_attribute_spec.rb b/spec/esse/integrations/elasticsearch-7/repository/documents_update_documents_attribute_spec.rb new file mode 100644 index 0000000..14437e1 --- /dev/null +++ b/spec/esse/integrations/elasticsearch-7/repository/documents_update_documents_attribute_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'support/shared_examples/repository_documents_update_documents_attribute' + +stack_describe 'elasticsearch', '7.x', Esse::Repository, '.update_documents_attribute' do + include_examples 'repository.update_documents_attribute' +end diff --git a/spec/esse/integrations/elasticsearch-8/repository/documents_update_documents_attribute_spec.rb b/spec/esse/integrations/elasticsearch-8/repository/documents_update_documents_attribute_spec.rb new file mode 100644 index 0000000..4ce3d0f --- /dev/null +++ b/spec/esse/integrations/elasticsearch-8/repository/documents_update_documents_attribute_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'support/shared_examples/repository_documents_update_documents_attribute' + +stack_describe 'elasticsearch', '8.x', Esse::Repository, '.update_documents_attribute' do + include_examples 'repository.update_documents_attribute' +end diff --git a/spec/esse/lazzy_document_header_spec.rb b/spec/esse/lazzy_document_header_spec.rb index c3c24da..6e18484 100644 --- a/spec/esse/lazzy_document_header_spec.rb +++ b/spec/esse/lazzy_document_header_spec.rb @@ -63,7 +63,7 @@ end context 'when _routing is present' do - let(:object) { { _routing: 'foo' } } + let(:object) { { routing: 'foo' } } it 'returns the _routing' do expect(doc.routing).to eq('foo') @@ -94,8 +94,8 @@ end end - context 'when _routing is present' do - let(:object) { { _routing: 'foo' } } + context 'when routing is present' do + let(:object) { { routing: 'foo' } } it 'returns the object' do expect(doc.to_h).to eq(object) @@ -166,6 +166,10 @@ expect(described_class.coerce_each([{_id: 1}])).to all(be_a(described_class)) end + it 'returns an array with a LazyDocumentHeader instance with the given Hash' do + expect(described_class.coerce_each(_id: 1)).to all(be_a(described_class)) + end + it 'removes invalid instances' do expect(described_class.coerce_each([nil, {_id: 1}, {}]).size).to eq(1) end diff --git a/spec/esse/primitives/array_utils_spec.rb b/spec/esse/primitives/array_utils_spec.rb new file mode 100644 index 0000000..3df36ac --- /dev/null +++ b/spec/esse/primitives/array_utils_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Esse::ArrayUtils do + describe '.wrap' do + specify do + expect(described_class.wrap(nil)).to eq([]) + end + + specify do + expect(described_class.wrap('a')).to eq(['a']) + end + + specify do + expect(described_class.wrap(['a'])).to eq(['a']) + end + + specify do + expect(described_class.wrap(%w[a b])).to eq(%w[a b]) + end + + specify do + expect(described_class.wrap({a: :b}).to_a).to eq([{a: :b}]) + end + end +end diff --git a/spec/esse/repository/document_spec.rb b/spec/esse/repository/document_spec.rb index d7f834b..f68881f 100644 --- a/spec/esse/repository/document_spec.rb +++ b/spec/esse/repository/document_spec.rb @@ -153,6 +153,40 @@ ) end end + + context 'with lazy_load_attributes' do + include_context 'with stories index definition' + + it 'yields serialized objects with lazy attributes when passing lazy_attributes: true' do + expected_data = [] + expect { + StoriesIndex::Story.each_serialized_batch(lazy_attributes: true) do |batch| + expected_data.push(*batch) + end + }.not_to raise_error + expect(expected_data.select { |doc| doc.to_h.key?(:tags) && doc.to_h.key?(:tags_count) }).not_to be_empty + end + + it 'yields serialized objects without lazy attributes when passing lazy_attributes: false' do + expected_data = [] + expect { + StoriesIndex::Story.each_serialized_batch(lazy_attributes: false) do |batch| + expected_data.push(*batch) + end + }.not_to raise_error + expect(expected_data.select { |doc| doc.to_h.key?(:tags) || doc.to_h.key?(:tags_count) }).to be_empty + end + + it 'yields serialized objects with lazy attributes when passing specific attributes' do + expected_data = [] + expect { + StoriesIndex::Story.each_serialized_batch(lazy_attributes: %i[tags]) do |batch| + expected_data.push(*batch) + end + }.not_to raise_error + expect(expected_data.select { |doc| doc.to_h.key?(:tags) && !doc.to_h.key?(:tags_count) }).not_to be_empty + end + end end describe '.documents' do @@ -391,7 +425,7 @@ def call end it 'returns false' do - expect(repo.lazy_document_attribute?(:foo)).to eq(false) + expect(repo.send(:lazy_document_attribute?, :foo)).to eq(false) end context 'with a lazy attribute' do @@ -401,10 +435,10 @@ def call end end - it 'returns true for both symbol and string' do - expect(repo.lazy_document_attribute?(:foo)).to eq(true) - expect(repo.lazy_document_attribute?('foo')).to eq(true) - expect(repo.lazy_document_attribute?(:bar)).to eq(false) + it 'returns attribute as it is defined' do + expect(repo.send(:lazy_document_attribute?, :foo)).to eq(true) + expect(repo.send(:lazy_document_attribute?, 'foo')).to eq(false) + expect(repo.send(:lazy_document_attribute?, :bar)).to eq(false) end end end @@ -431,11 +465,11 @@ def call end it 'defines a lazy attribute' do - expect(repo.lazy_document_attributes["foo"]).to match_array([ + expect(repo.lazy_document_attributes[:foo]).to match_array([ be < Esse::DocumentLazyAttribute, {}, ]) - expect(repo.lazy_document_attribute?(:foo)).to eq(true) + expect(repo.send(:lazy_document_attribute?, :foo)).to eq(true) end end @@ -470,11 +504,11 @@ def call end it 'defines a lazy attribute' do - expect(repo.lazy_document_attributes["foo"]).to match_array([ + expect(repo.lazy_document_attributes[:foo]).to match_array([ TheFooParser, {}, ]) - expect(repo.lazy_document_attribute?(:foo)).to eq(true) + expect(repo.send(:lazy_document_attribute?, :foo)).to eq(true) expect(repo.lazy_document_attributes).to be_frozen end end @@ -487,11 +521,11 @@ def call end it 'defines a lazy attribute' do - expect(repo.lazy_document_attributes["foo"]).to match_array([ + expect(repo.lazy_document_attributes[:foo]).to match_array([ be < Esse::DocumentLazyAttribute, {}, ]) - expect(repo.lazy_document_attribute?(:foo)).to eq(true) + expect(repo.send(:lazy_document_attribute?, :foo)).to eq(true) expect(repo.lazy_document_attributes).to be_frozen end end @@ -504,11 +538,11 @@ def call end it 'defines a lazy attribute' do - expect(repo.lazy_document_attributes["foo"]).to match_array([ + expect(repo.lazy_document_attributes[:foo]).to match_array([ be < Esse::DocumentLazyAttribute, { bar: 'baz' }, ]) - expect(repo.lazy_document_attribute?(:foo)).to eq(true) + expect(repo.send(:lazy_document_attribute?, :foo)).to eq(true) expect(repo.lazy_document_attributes).to be_frozen end end diff --git a/spec/esse/repository/lazy_document_spec.rb b/spec/esse/repository/lazy_document_spec.rb index 27ce756..5734f6a 100644 --- a/spec/esse/repository/lazy_document_spec.rb +++ b/spec/esse/repository/lazy_document_spec.rb @@ -12,7 +12,7 @@ end it 'returns all the lazy document attribute names as default' do - expect(repo.lazy_document_attribute_names).to eq(%w[foo bar]) + expect(repo.lazy_document_attribute_names).to eq(%i[foo bar]) end it 'returns an empty array when no lazy document attributes are defined' do @@ -20,7 +20,7 @@ end it 'returs all the lazy document attribute names when passing true' do - expect(repo.lazy_document_attribute_names(true)).to eq(%w[foo bar]) + expect(repo.lazy_document_attribute_names(true)).to eq(%i[foo bar]) end it 'returns an empty array when passing false' do @@ -28,15 +28,15 @@ end it 'returns an array of lazy document attribute names when passing an array of names' do - expect(repo.lazy_document_attribute_names(%w[foo])).to eq(%w[foo]) + expect(repo.lazy_document_attribute_names(%w[foo])).to eq(%i[foo]) end it 'returns an array of lazy document attribute names when passing a single name' do - expect(repo.lazy_document_attribute_names('foo')).to eq(%w[foo]) + expect(repo.lazy_document_attribute_names('foo')).to eq(%i[foo]) end it 'returns an array of lazy document attribute names when passing a single name as symbol' do - expect(repo.lazy_document_attribute_names(:foo)).to eq(%w[foo]) + expect(repo.lazy_document_attribute_names(:foo)).to eq(%i[foo]) end end @@ -59,7 +59,7 @@ end it 'returns an empty array when no ids are provided' do - expect(repo.documents_for_lazy_attribute(:city_names)).to eq([]) + expect(repo.documents_for_lazy_attribute(:city_names, nil)).to eq([]) end it 'returns an empty array when no ids are found' do @@ -80,7 +80,7 @@ end it 'returns an empty array when no ids are provided' do - expect(repo.documents_for_lazy_attribute(:city_names)).to eq([]) + expect(repo.documents_for_lazy_attribute(:city_names, nil)).to eq([]) end it 'returns an empty array when no ids are found' do @@ -115,7 +115,7 @@ end it 'returns an empty array when no ids are provided' do - expect(repo.documents_for_lazy_attribute(:city_names)).to eq([]) + expect(repo.documents_for_lazy_attribute(:city_names, nil)).to eq([]) end it 'returns an empty array when no ids are found' do @@ -137,7 +137,7 @@ end it 'do not include duplicate documents' do - docs = repo.documents_for_lazy_attribute(:city_names, '2', '2', Esse::LazyDocumentHeader.coerce(id: '2')) + docs = repo.documents_for_lazy_attribute(:city_names, ['2', '2', Esse::LazyDocumentHeader.coerce(id: '2')]) expect(docs).to eq([ Esse::HashDocument.new(_id: '2', city_names: 'London') ]) @@ -159,7 +159,7 @@ end it 'returns an array of documents that match with the provided ids' do - docs = repo.documents_for_lazy_attribute(:city_names, '2', '3', '4') + docs = repo.documents_for_lazy_attribute(:city_names, ['2', '3', '4']) expect(docs).to eq([ Esse::HashDocument.new(_id: '2', city_names: 'London'), Esse::HashDocument.new(_id: '3', city_names: nil), diff --git a/spec/support/shared_contexts/geos_index_definition.rb b/spec/support/shared_contexts/geos_index_definition.rb index d258844..5a22722 100644 --- a/spec/support/shared_contexts/geos_index_definition.rb +++ b/spec/support/shared_contexts/geos_index_definition.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# rubocop:disable RSpec/MultipleMemoizedHelpers RSpec.shared_context 'with geos index definition' do let(:states_batches) do [ @@ -92,3 +93,4 @@ def source end end end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/spec/support/shared_contexts/stories_index_definition.rb b/spec/support/shared_contexts/stories_index_definition.rb new file mode 100644 index 0000000..f8dd186 --- /dev/null +++ b/spec/support/shared_contexts/stories_index_definition.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# rubocop:disable RSpec/MultipleMemoizedHelpers +RSpec.shared_context 'with stories index definition' do + let(:nyt) do + { id: 1, name: 'The New York Times', slug: 'nyt' } + end + let(:wsj) do + { id: 2, name: 'The Wall Street Journal', slug: 'wsj' } + end + let(:nyt_stories) do + [ + { id: 1_001, title: 'The first story', story: nyt, published_at: '2019-01-01', tags: %w[news politics], publication: 'nyt' }, + { id: 1_002, title: 'The second story', story: nyt, published_at: '2019-01-02', tags: %w[news], publication: 'nyt' }, + { id: 1_003, title: 'The third story', story: nyt, published_at: nil, tags: %w[news politics], publication: 'nyt' }, + ] + end + let(:wsj_stories) do + [ + { id: 2_001, title: 'The first story', story: wsj, published_at: '2019-01-01', tags: %w[news politics], publication: 'wsj' }, + { id: 2_002, title: 'The second story', story: wsj, published_at: '2019-01-01', tags: %w[news], publication: 'wsj' }, + { id: 2_003, title: 'The third story', story: wsj, published_at: nil, tags: %w[news politics], publication: 'wsj' }, + ] + end + let(:publication_stories) do + { + nyt => nyt_stories, + wsj => wsj_stories, + } + end + let(:stories) { publication_stories.values.flatten } + + before do + # closure for the stub_index block + ds = stories + + stub_index(:stories) do + repository :story do + collection do |**context, &block| + stories = context[:conditions] ? ds.select(&context[:conditions]) : ds + block.call(stories, **context) unless stories.empty? + end + document do |story, **context| + { + _id: story[:id], + _routing: story[:publication], + publication: story[:publication], + title: story[:title], + published_at: story[:published_at], + } + end + lazy_document_attribute :tags do |docs| + docs.map do |doc| + [doc, ds.find { |s| s[:id] == doc.id.to_i }&.[](:tags) || []] + end.to_h + end + lazy_document_attribute :tags_count do |docs| + docs.map do |doc| + [doc, (ds.find { |s| s[:id] == doc.id.to_i }&.[](:tags) || []).size] + end.to_h + end + end + end + end +end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/spec/support/shared_examples/repository_documents_import.rb b/spec/support/shared_examples/repository_documents_import.rb index 8bad72e..5139602 100644 --- a/spec/support/shared_examples/repository_documents_import.rb +++ b/spec/support/shared_examples/repository_documents_import.rb @@ -142,4 +142,48 @@ end end end + + context 'when the document routing is set' do + include_context 'with stories index definition' + + it 'indexes the data and bulk updates all the document routing' do |example| + es_client do |client, _conf, cluster| + StoriesIndex.create_index(alias: true) + + resp = nil + expect { + resp = StoriesIndex::Story.import + }.not_to raise_error + expect(resp).to eq(stories.size) + + StoriesIndex.refresh + expect(StoriesIndex.count).to eq(stories.size) + + doc = StoriesIndex.get(id: '1001', routing: 'nyt') + expect(doc.dig('_source', 'publication')).to eq('nyt') + expect(doc.dig('_source', 'tags')).to be(nil) + unless %w[1.x].include?(example.metadata[:es_version]) + expect(doc.dig('_routing')).to eq('nyt') + end + end + end + + it 'lazy update the document tags attribute' do + es_client do |client, _conf, cluster| + StoriesIndex.create_index(alias: true) + + resp = nil + expect { + resp = StoriesIndex::Story.import(lazy_update_document_attributes: %i[tags]) + }.not_to raise_error + expect(resp).to eq(stories.size) + + StoriesIndex.refresh + expect(StoriesIndex.count).to eq(stories.size) + + doc = StoriesIndex.get(id: '1001', routing: 'nyt') + expect(doc.dig('_source', 'tags')).to eq(%w[news politics]) + end + end + end end diff --git a/spec/support/shared_examples/repository_documents_update_documents_attribute.rb b/spec/support/shared_examples/repository_documents_update_documents_attribute.rb new file mode 100644 index 0000000..d076dc5 --- /dev/null +++ b/spec/support/shared_examples/repository_documents_update_documents_attribute.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +# rubocop:disable RSpec/MultipleExpectations +RSpec.shared_examples 'repository.update_documents_attribute' do + context 'when the index has routing' do + include_context 'with stories index definition' + + it 'adds the :tags from lazy_document_attribute to the document' do + es_client do |client, _conf, cluster| + StoriesIndex.create_index(alias: true) + + resp = nil + expect { + resp = StoriesIndex::Story.import(context: { conditions: ->(s) { s[:publication] == 'nyt' } }) + }.not_to raise_error + expect(resp).to eq(nyt_stories.size) + + StoriesIndex.refresh + expect(StoriesIndex.count).to eq(nyt_stories.size) + + doc = StoriesIndex.get(id: '1001', routing: 'nyt') + expect(doc.dig('_source', 'publication')).to eq('nyt') + expect(doc.dig('_source', 'tags')).to be(nil) + + expect { + resp = StoriesIndex::Story.update_documents_attribute(:tags, { _id: '1001', routing: 'nyt' }, refresh: true) + }.not_to raise_error + + doc = StoriesIndex.get(id: '1001', routing: 'nyt') + expect(doc.dig('_source', 'publication')).to eq('nyt') + expect(doc.dig('_source', 'tags')).to eq(%w[news politics]) + end + end + end + + context 'when the index does not have routing' do + include_context 'with geos index definition' + + it 'adds the :location from lazy_document_attribute to the document' do + es_client do |client, _conf, cluster| + GeosIndex.create_index + + resp = nil + expect { + resp = GeosIndex::County.import + }.not_to raise_error + expect(resp).to eq(total_counties) + + GeosIndex.refresh + expect(GeosIndex.count).to eq(total_counties) + + doc = GeosIndex.get(id: '999') + expect(doc.dig('_source', 'country')).to be(nil) + + expect { + resp = GeosIndex::County.update_documents_attribute(:country, '999', refresh: true) + }.not_to raise_error + + doc = GeosIndex.get(id: '999') + expect(doc.dig('_source', 'country')).to eq('US') + end + end + end +end