Charlie GunyonandCharlie Gunyon
311494e86e
* Add stub source and elastic API funcs * Spawn workers and ship chunks * Now successfully detects a credential - Added tests - Added some documentation comments - Threaded the passed context through to all the API requests * Linting fixes * Add integration tests and resolve some bugs they uncovered * Logstash -> Elasticsearch * Add support for --index-pattern * Add support for --query-json * Use structs instead of string building to construct a search body * Support --since-timestamp * Implement additional authentication methods * Fix some small bugs * Refactoring to support --best-effort-scan * Finish implementation of --best-effort-scan * Implement scan catch-up * Finish connecting support for nodes CLI arg * Add some integration tests around the catchup mechanism * go mod tidy * Fix some linting issues * Remove some debugging Prints * Move off of _doc * Remove informational Printf and add informational logging * Remove debugging logging * Copy the index from the outer loop as well * Don't burn up the ES API with rapid requests if there's no work to do in subsequent scans * No need to export UnitOfWork.AddSearch * Use a better name for the range query variable when building the timestamp range clause in searches * Replace some unlocking defers with explicit unlocks to make the synchronized part of the code clearer * found -> ok * Remove superfluous buildElasticClient method --------- Co-authored-by: Charlie Gunyon <[email protected]>
141 lines
5.5 KiB
Go
141 lines
5.5 KiB
Go
package elasticsearch
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestSource_distributeDocumentScans(t *testing.T) {
|
|
indices := Indices{
|
|
indices: []*Index{
|
|
&Index{name: "index", documentCount: 20},
|
|
&Index{name: "index2", documentCount: 9},
|
|
&Index{name: "index3", documentCount: 0},
|
|
},
|
|
filterParams: &FilterParams{},
|
|
}
|
|
|
|
t.Run(
|
|
"Distributing 30 documents from 3 indices (1 empty) with 2 workers works",
|
|
func(t *testing.T) {
|
|
uows := distributeDocumentScans(&indices, 2, .9)
|
|
|
|
assert.Equal(t, 2, len(uows))
|
|
|
|
assert.Equal(t, 14, uows[0].maxDocumentCount)
|
|
assert.Equal(t, 14, uows[0].documentCount)
|
|
assert.Equal(t, 1, len(uows[0].documentSearches))
|
|
|
|
assert.Equal(t, "index", uows[0].documentSearches[0].index.name)
|
|
assert.Equal(t, 0, uows[0].documentSearches[0].offset)
|
|
assert.Equal(t, 14, uows[0].documentSearches[0].documentCount)
|
|
assert.Equal(t, 1, uows[0].documentSearches[0].skipCount)
|
|
|
|
assert.Equal(t, 15, uows[1].maxDocumentCount)
|
|
assert.Equal(t, 15, uows[1].documentCount)
|
|
assert.Equal(t, 2, len(uows[1].documentSearches))
|
|
|
|
assert.Equal(t, "index", uows[1].documentSearches[0].index.name)
|
|
assert.Equal(t, 14, uows[1].documentSearches[0].offset)
|
|
assert.Equal(t, 6, uows[1].documentSearches[0].documentCount)
|
|
assert.Equal(t, 0, uows[1].documentSearches[0].skipCount)
|
|
|
|
assert.Equal(t, "index2", uows[1].documentSearches[1].index.name)
|
|
assert.Equal(t, 0, uows[1].documentSearches[1].offset)
|
|
assert.Equal(t, 9, uows[1].documentSearches[1].documentCount)
|
|
assert.Equal(t, 0, uows[1].documentSearches[1].skipCount)
|
|
},
|
|
)
|
|
}
|
|
|
|
func TestSource_addSearch(t *testing.T) {
|
|
index := Index{name: "index1", documentCount: 20}
|
|
index2 := Index{name: "index2", documentCount: 10}
|
|
index3 := Index{name: "index3", documentCount: 0}
|
|
|
|
uow := NewUnitOfWork(10)
|
|
|
|
// Does filling up a UOW with a larger index work?
|
|
offset := uow.addSearch(&index, &FilterParams{}, 0, 1.0)
|
|
assert.Equal(t, 10, offset)
|
|
assert.Equal(t, 10, uow.maxDocumentCount)
|
|
assert.Equal(t, uow.maxDocumentCount, uow.documentCount)
|
|
assert.Equal(t, 1, len(uow.documentSearches))
|
|
assert.Equal(t, index.name, uow.documentSearches[0].index.name)
|
|
assert.Equal(t, 0, uow.documentSearches[0].offset)
|
|
assert.Equal(t, uow.maxDocumentCount, uow.documentSearches[0].documentCount)
|
|
|
|
// Does trying to add another range into a full UOW leave it unchanged?
|
|
offset2 := uow.addSearch(&index2, &FilterParams{}, 0, 1.0)
|
|
assert.Equal(t, 0, offset2)
|
|
assert.Equal(t, 10, uow.maxDocumentCount)
|
|
assert.Equal(t, uow.maxDocumentCount, uow.documentCount)
|
|
assert.Equal(t, 1, len(uow.documentSearches))
|
|
assert.Equal(t, index.name, uow.documentSearches[0].index.name)
|
|
assert.Equal(t, 0, uow.documentSearches[0].offset)
|
|
assert.Equal(t, uow.maxDocumentCount, uow.documentSearches[0].documentCount)
|
|
|
|
// Does trying to add an index with no documents leave it unchanged?
|
|
offset += uow.addSearch(&index3, &FilterParams{}, 0, 1.0)
|
|
assert.Equal(t, 10, offset)
|
|
assert.Equal(t, 10, uow.maxDocumentCount)
|
|
assert.Equal(t, uow.maxDocumentCount, uow.documentCount)
|
|
assert.Equal(t, 1, len(uow.documentSearches))
|
|
assert.Equal(t, index.name, uow.documentSearches[0].index.name)
|
|
assert.Equal(t, 0, uow.documentSearches[0].offset)
|
|
assert.Equal(t, uow.maxDocumentCount, uow.documentSearches[0].documentCount)
|
|
|
|
// Does filling up another UOW with a larger index work?
|
|
uow2 := NewUnitOfWork(9)
|
|
|
|
offset += uow2.addSearch(&index, &FilterParams{}, offset, 1.0)
|
|
assert.Equal(t, 19, offset)
|
|
assert.Equal(t, 9, uow2.maxDocumentCount)
|
|
assert.Equal(t, uow2.maxDocumentCount, uow2.documentCount)
|
|
assert.Equal(t, 1, len(uow2.documentSearches))
|
|
assert.Equal(t, index.name, uow2.documentSearches[0].index.name)
|
|
assert.Equal(t, 10, uow2.documentSearches[0].offset)
|
|
assert.Equal(t, uow2.maxDocumentCount, uow2.documentSearches[0].documentCount)
|
|
|
|
// Does finishing off an index into a UOW with room to spare work?
|
|
uow3 := NewUnitOfWork(9)
|
|
|
|
offset += uow3.addSearch(&index, &FilterParams{}, offset, 1.0)
|
|
assert.Equal(t, 20, offset)
|
|
assert.Equal(t, 9, uow3.maxDocumentCount)
|
|
assert.Equal(t, 1, uow3.documentCount)
|
|
assert.Equal(t, 1, len(uow3.documentSearches))
|
|
assert.Equal(t, index.name, uow3.documentSearches[0].index.name)
|
|
assert.Equal(t, 19, uow3.documentSearches[0].offset)
|
|
assert.Equal(t, 1, uow3.documentSearches[0].documentCount)
|
|
|
|
uow = NewUnitOfWork(21)
|
|
|
|
// Does adding an empty range into a new UOW leave it unchanged?
|
|
offset = uow.addSearch(&index3, &FilterParams{}, 0, 1.0)
|
|
assert.Equal(t, 0, offset)
|
|
assert.Equal(t, 21, uow.maxDocumentCount)
|
|
assert.Equal(t, 0, uow.documentCount)
|
|
assert.Equal(t, 0, len(uow.documentSearches))
|
|
|
|
// Does adding a range into a larger UOW work?
|
|
offset = uow.addSearch(&index, &FilterParams{}, 0, 1.0)
|
|
assert.Equal(t, 20, offset)
|
|
assert.Equal(t, 1, len(uow.documentSearches))
|
|
assert.Equal(t, index.name, uow.documentSearches[0].index.name)
|
|
assert.Equal(t, 0, uow.documentSearches[0].offset)
|
|
assert.Equal(t, 20, uow.documentSearches[0].documentCount)
|
|
|
|
// Does filling up a UOW that already has a range in it work?
|
|
offset = uow.addSearch(&index2, &FilterParams{}, 0, 1.0)
|
|
assert.Equal(t, 1, offset)
|
|
assert.Equal(t, 2, len(uow.documentSearches))
|
|
assert.Equal(t, index.name, uow.documentSearches[0].index.name)
|
|
assert.Equal(t, 0, uow.documentSearches[0].offset)
|
|
assert.Equal(t, 20, uow.documentSearches[0].documentCount)
|
|
assert.Equal(t, index2.name, uow.documentSearches[1].index.name)
|
|
assert.Equal(t, 0, uow.documentSearches[1].offset)
|
|
assert.Equal(t, 1, uow.documentSearches[1].documentCount)
|
|
}
|