From 709f90dc6804c41913fbbda92cc0a70a71c06e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Sz=C3=BCcs?= <127139797+balazs-szucs@users.noreply.github.com> Date: Fri, 16 Jan 2026 03:56:45 +0100 Subject: [PATCH] fix(parsers): increase request interval to prevent rate limiting issues (#2282) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Balázs Szücs --- .../metadata/parser/ComicvineBookParser.java | 2 +- .../metadata/parser/GoodReadsParser.java | 2 +- .../service/metadata/parser/GoogleParser.java | 20 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/ComicvineBookParser.java b/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/ComicvineBookParser.java index 0d876a340..fa2767d0d 100644 --- a/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/ComicvineBookParser.java +++ b/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/ComicvineBookParser.java @@ -46,7 +46,7 @@ public class ComicvineBookParser implements BookParser { private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); private static final Pattern SPECIAL_ISSUE_PATTERN = Pattern.compile("(annual|special|one-?shot)\\s+(\\d+)", Pattern.CASE_INSENSITIVE); private static final Pattern YEAR_PATTERN = Pattern.compile("\\(?(\\d{4})\\)?"); - private static final long MIN_REQUEST_INTERVAL_MS = 1000; + private static final long MIN_REQUEST_INTERVAL_MS = 2000; private final ObjectMapper objectMapper; private final AppSettingService appSettingService; diff --git a/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoodReadsParser.java b/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoodReadsParser.java index 1baad5322..13d2ee5e0 100644 --- a/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoodReadsParser.java +++ b/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoodReadsParser.java @@ -133,7 +133,7 @@ public class GoodReadsParser implements BookParser { if (detailedMetadata != null) { fetchedMetadata.add(detailedMetadata); } - Thread.sleep(Duration.ofSeconds(1)); + Thread.sleep(Duration.ofSeconds(2)); } catch (Exception e) { log.error("Error fetching metadata for book: {}", preview.getGoodreadsId(), e); } diff --git a/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoogleParser.java b/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoogleParser.java index 50df3476e..01e2a2719 100644 --- a/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoogleParser.java +++ b/booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoogleParser.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -35,10 +36,12 @@ public class GoogleParser implements BookParser { private static final Pattern FOUR_DIGIT_YEAR_PATTERN = Pattern.compile("\\d{4}"); private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); private static final Pattern SPECIAL_CHARACTERS_PATTERN = Pattern.compile("[.,\\-\\[\\]{}()!@#$%^&*_=+|~`<>?/\";:]"); + private static final long MIN_REQUEST_INTERVAL_MS = 1500; private final ObjectMapper objectMapper; private final AppSettingService appSettingService; private final HttpClient httpClient = HttpClient.newHttpClient(); private static final String GOOGLE_BOOKS_API_URL = "https://www.googleapis.com/books/v1/volumes"; + private final AtomicLong lastRequestTime = new AtomicLong(0); @Override public BookMetadata fetchTopMetadata(Book book, FetchMetadataRequest fetchMetadataRequest) { @@ -57,6 +60,8 @@ public class GoogleParser implements BookParser { private List getMetadataListByIsbn(String isbn) { try { + waitForRateLimit(); + URI uri = UriComponentsBuilder.fromUriString(getApiUrl()) .queryParam("q", "isbn:" + isbn.replace("-", "")) .build() @@ -86,6 +91,8 @@ public class GoogleParser implements BookParser { public List getMetadataListByTerm(String term) { try { + waitForRateLimit(); + URI uri = UriComponentsBuilder.fromUriString(getApiUrl()) .queryParam("q", term) .build() @@ -225,4 +232,17 @@ public class GoogleParser implements BookParser { .toUri() .toString(); } + + private void waitForRateLimit() { + long now = System.currentTimeMillis(); + long timeSinceLastRequest = now - lastRequestTime.get(); + if (timeSinceLastRequest < MIN_REQUEST_INTERVAL_MS) { + try { + Thread.sleep(MIN_REQUEST_INTERVAL_MS - timeSinceLastRequest); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + lastRequestTime.set(System.currentTimeMillis()); + } } \ No newline at end of file