booklore-app/booklore
1
0
Fork 0
mirror of https://github.com/booklore-app/booklore.git synced 2026-02-18 00:17:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(parsers): increase request interval to prevent rate limiting issues (#2282)

Browse Source 
Signed-off-by: Balázs Szücs <bszucs1209@gmail.com>
This commit is contained in:
Balázs Szücs
2026-01-16 03:56:45 +01:00
committed by GitHub
parent 574c01b412
commit 709f90dc68
3 changed files with 22 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/ComicvineBookParser.java
View File

@@ -46,7 +46,7 @@ public class ComicvineBookParser implements BookParser {
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
private static final Pattern SPECIAL_ISSUE_PATTERN = Pattern.compile("(annual|special|one-?shot)\\s+(\\d+)", Pattern.CASE_INSENSITIVE);
private static final Pattern YEAR_PATTERN = Pattern.compile("\\(?(\\d{4})\\)?");
private static final long MIN_REQUEST_INTERVAL_MS = 1000;
private static final long MIN_REQUEST_INTERVAL_MS = 2000;
private final ObjectMapper objectMapper;
private final AppSettingService appSettingService;

2
booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoodReadsParser.java
View File

@@ -133,7 +133,7 @@ public class GoodReadsParser implements BookParser {
if (detailedMetadata != null) {
fetchedMetadata.add(detailedMetadata);
}
Thread.sleep(Duration.ofSeconds(1));
Thread.sleep(Duration.ofSeconds(2));
} catch (Exception e) {
log.error("Error fetching metadata for book: {}", preview.getGoodreadsId(), e);
}

20
booklore-api/src/main/java/com/adityachandel/booklore/service/metadata/parser/GoogleParser.java
View File

@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -35,10 +36,12 @@ public class GoogleParser implements BookParser {
private static final Pattern FOUR_DIGIT_YEAR_PATTERN = Pattern.compile("\\d{4}");
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
private static final Pattern SPECIAL_CHARACTERS_PATTERN = Pattern.compile("[.,\\-\\[\\]{}()!@#$%^&*_=+|~`<>?/\";:]");
private static final long MIN_REQUEST_INTERVAL_MS = 1500;
private final ObjectMapper objectMapper;
private final AppSettingService appSettingService;
private final HttpClient httpClient = HttpClient.newHttpClient();
private static final String GOOGLE_BOOKS_API_URL = "https://www.googleapis.com/books/v1/volumes";
private final AtomicLong lastRequestTime = new AtomicLong(0);
@Override
public BookMetadata fetchTopMetadata(Book book, FetchMetadataRequest fetchMetadataRequest) {
@@ -57,6 +60,8 @@ public class GoogleParser implements BookParser {
private List<BookMetadata> getMetadataListByIsbn(String isbn) {
try {
waitForRateLimit();
URI uri = UriComponentsBuilder.fromUriString(getApiUrl())
.queryParam("q", "isbn:" + isbn.replace("-", ""))
.build()
@@ -86,6 +91,8 @@ public class GoogleParser implements BookParser {
public List<BookMetadata> getMetadataListByTerm(String term) {
try {
waitForRateLimit();
URI uri = UriComponentsBuilder.fromUriString(getApiUrl())
.queryParam("q", term)
.build()
@@ -225,4 +232,17 @@ public class GoogleParser implements BookParser {
.toUri()
.toString();
}
private void waitForRateLimit() {
long now = System.currentTimeMillis();
long timeSinceLastRequest = now - lastRequestTime.get();
if (timeSinceLastRequest < MIN_REQUEST_INTERVAL_MS) {
try {
Thread.sleep(MIN_REQUEST_INTERVAL_MS - timeSinceLastRequest);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
lastRequestTime.set(System.currentTimeMillis());
}
}