From 0613e120a78a88b9c32bc75e079d7a30f81f1e4d Mon Sep 17 00:00:00 2001 From: Keith Dreibelbis Date: Wed, 23 Oct 2019 13:32:00 -0700 Subject: [PATCH] Allow dispatch caller to get stats back --- src/main/java/org/dataloader/DataLoader.java | 13 ++++++ .../java/org/dataloader/DataLoaderHelper.java | 22 +++++++--- .../org/dataloader/DataLoaderRegistry.java | 40 ++++++++++++++----- .../java/org/dataloader/DataLoaderTest.java | 27 ++++++++++++- 4 files changed, 87 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index a17b347d..cef34764 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -470,6 +470,19 @@ public CompletableFuture> loadMany(List keys, List keyContext * @return the promise of the queued load requests */ public CompletableFuture> dispatch() { + return helper.dispatch().futureList; + } + + /** + * Dispatches the queued load requests to the batch execution function and returns both the promise of the result + * and the number of entries that were dispatched. + *

+ * If batching is disabled, or there are no queued requests, then a succeeded promise with no entries dispatched is + * returned. + * + * @return the promise of the queued load requests and the number of entries dispatched. + */ + public DataLoaderHelper.DispatchResult dispatchWithCounts() { return helper.dispatch(); } diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index d86c3fb9..d3fbd84c 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -134,7 +134,16 @@ Object getCacheKey(K key) { loaderOptions.cacheKeyFunction().get().getKey(key) : key; } - CompletableFuture> dispatch() { + public static class DispatchResult { + public final CompletableFuture> futureList; + public final int totalEntriesHandled; + public DispatchResult(CompletableFuture> futureList, int totalEntriesHandled) { + this.futureList = futureList; + this.totalEntriesHandled = totalEntriesHandled; + } + } + + DispatchResult dispatch() { boolean batchingEnabled = loaderOptions.batchingEnabled(); // // we copy the pre-loaded set of futures ready for dispatch @@ -149,9 +158,10 @@ CompletableFuture> dispatch() { }); loaderQueue.clear(); } - if (!batchingEnabled || keys.size() == 0) { - return CompletableFuture.completedFuture(emptyList()); + if (!batchingEnabled || keys.isEmpty()) { + return new DispatchResult(CompletableFuture.completedFuture(emptyList()), 0); } + final int totalEntriesHandled = keys.size(); // // order of keys -> values matter in data loader hence the use of linked hash map // @@ -164,11 +174,13 @@ CompletableFuture> dispatch() { // via calls to load("foo") and loadMany(["foo","bar"]) // int maxBatchSize = loaderOptions.maxBatchSize(); + CompletableFuture> futureList; if (maxBatchSize > 0 && maxBatchSize < keys.size()) { - return sliceIntoBatchesOfBatches(keys, queuedFutures, callContexts, maxBatchSize); + futureList = sliceIntoBatchesOfBatches(keys, queuedFutures, callContexts, maxBatchSize); } else { - return dispatchQueueBatch(keys, callContexts, queuedFutures); + futureList = dispatchQueueBatch(keys, callContexts, queuedFutures); } + return new DispatchResult(futureList, totalEntriesHandled); } private CompletableFuture> sliceIntoBatchesOfBatches(List keys, List> queuedFutures, List callContexts, int maxBatchSize) { diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 79067e96..3b0e719c 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -7,6 +7,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; + import org.dataloader.stats.Statistics; /** @@ -22,7 +23,6 @@ public class DataLoaderRegistry { * * @param key the key to put the data loader under * @param dataLoader the data loader to register - * * @return this registry */ public DataLoaderRegistry register(String key, DataLoader dataLoader) { @@ -33,15 +33,14 @@ public DataLoaderRegistry register(String key, DataLoader dataLoader) { /** * Computes a data loader if absent or return it if it was * already registered at that key. - * + *

* Note: The entire method invocation is performed atomically, * so the function is applied at most once per key. * - * @param key the key of the data loader + * @param key the key of the data loader * @param mappingFunction the function to compute a data loader - * @param the type of keys - * @param the type of values - * + * @param the type of keys + * @param the type of values * @return a data loader */ @SuppressWarnings("unchecked") @@ -55,7 +54,6 @@ public DataLoader computeIfAbsent(final String key, * and return a new combined registry * * @param registry the registry to combine into this registry - * * @return a new combined registry */ public DataLoaderRegistry combine(DataLoaderRegistry registry) { @@ -77,7 +75,6 @@ public DataLoaderRegistry combine(DataLoaderRegistry registry) { * This will unregister a new dataloader * * @param key the key of the data loader to unregister - * * @return this registry */ public DataLoaderRegistry unregister(String key) { @@ -91,7 +88,6 @@ public DataLoaderRegistry unregister(String key) { * @param key the key of the data loader * @param the type of keys * @param the type of values - * * @return a data loader or null if its not present */ @SuppressWarnings("unchecked") @@ -114,6 +110,32 @@ public void dispatchAll() { getDataLoaders().forEach(DataLoader::dispatch); } + /** + * Similar to {@link DataLoaderRegistry#dispatchAll()}, this calls {@link org.dataloader.DataLoader#dispatch()} on + * each of the registered {@link org.dataloader.DataLoader}s, but returns the number of dispatches. + * + * @return total number of entries that were dispatched from registered {@link org.dataloader.DataLoader}s. + */ + public int dispatchAllWithCount() { + int sum = 0; + for (DataLoader dataLoader : getDataLoaders()) { + sum += dataLoader.dispatchWithCounts().totalEntriesHandled; + } + return sum; + } + + /** + * @return The sum of all batched key loads that need to be dispatched from all registered + * {@link org.dataloader.DataLoader}s + */ + public int dispatchDepth() { + int totalDispatchDepth = 0; + for (DataLoader dataLoader : getDataLoaders()) { + totalDispatchDepth += dataLoader.dispatchDepth(); + } + return totalDispatchDepth; + } + /** * @return a combined set of statistics for all data loaders in this registry presented * as the sum of all their statistics diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index cf389859..6c4a07d7 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -101,6 +101,19 @@ public void should_Resolve_to_empty_list_when_no_keys_supplied() { assertThat(futureEmpty.join(), empty()); } + @Test + public void should_Return_zero_entries_dispatched_when_no_keys_supplied() { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = new DataLoader<>(keysAsValues()); + CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); + futureEmpty.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(0)); + success.set(true); + }); + DataLoaderHelper.DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); + await().untilAtomic(success, is(true)); + assertThat(dispatchResult.totalEntriesHandled, equalTo(0)); + } @Test public void should_Batch_multiple_requests() throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); @@ -113,7 +126,19 @@ public void should_Batch_multiple_requests() throws ExecutionException, Interrup await().until(() -> future1.isDone() && future2.isDone()); assertThat(future1.get(), equalTo(1)); assertThat(future2.get(), equalTo(2)); - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); + assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); } + + @Test + public void should_Return_number_of_batched_entries() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = identityLoader.load(1); + CompletableFuture future2 = identityLoader.load(2); + DataLoaderHelper.DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(dispatchResult.totalEntriesHandled, equalTo(2)); } @Test