Implemented incremental pagination when no images are valid
Unit Tests with docker compose / unit-test (push) Successful in 19s Details
Build image / build (push) Successful in 43s Details

This commit is contained in:
Sugui 2024-06-15 20:50:22 +02:00
parent dd8066207e
commit 588970e18c
2 changed files with 28 additions and 21 deletions

View File

@ -6,8 +6,9 @@ import GelbooruServiceResponse from "src/types/GelbooruServiceResponse";
export const LIMIT = env.GELBOORU_IMAGES_PER_REQUEST || 100;
export const TAGS = encodeURIComponent(env.GELBOORU_TAGS || "");
export async function get(): Promise<GelbooruServiceResponse> {
const url: string = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=${LIMIT}&json=1&tags=${TAGS}`;
export async function get(pid?: number): Promise<GelbooruServiceResponse> {
const pidQueryParam = pid ? `&pid=${pid}` : "";
const url: string = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=${LIMIT}&json=1&tags=${TAGS}${pidQueryParam}`;
const response: GelbooruApiResponse = (await fetch(url).then(
async (res) => {

View File

@ -2,22 +2,21 @@ import GelbooruServiceResponse from "src/types/GelbooruServiceResponse";
import Image from "src/types/Image";
import logger from "src/logger";
import { Mutex } from "async-mutex";
import * as gelbooruApiService from 'src/services/gelbooruApiService';
import * as botApiService from 'src/services/botApiService';
import * as gelbooruApiService from "src/services/gelbooruApiService";
import * as botApiService from "src/services/botApiService";
const mutex: Mutex = new Mutex();
const postsQueue: Image[] = [];
// We wrap the function into a Mutex because it's not thread-safe
export async function get(): Promise<Image> {
return await mutex.runExclusive(() => unsafeGet())
return await mutex.runExclusive(() => unsafeGet());
}
async function unsafeGet(): Promise<Image> {
while (postsQueue.length === 0) {
const validPosts = await getNewValidImages();
validPosts.map(post => postsQueue.push(post));
logger.info(`Got ${validPosts.length} images from remote`);
if (postsQueue.length === 0) {
const validPosts = await getNewImages();
validPosts.map((post) => postsQueue.push(post));
}
return popImage();
}
@ -25,27 +24,34 @@ async function unsafeGet(): Promise<Image> {
function popImage(): Image {
const image = postsQueue.pop();
if (image) {
return image
return image;
} else {
throw Error("Can't pop from an empty list");
}
}
async function getNewValidImages(): Promise<Image[]> {
/**
* Returns an array of images that are not present in the bot database
*/
async function getNewImages(): Promise<Image[]> {
for (let pageId = 0; ; pageId++) {
const gelbooruResponse: GelbooruServiceResponse =
await gelbooruApiService.get();
await gelbooruApiService.get(pageId);
const posts = gelbooruResponse.posts;
const botResponse = await botApiService.getAll();
const imagesUrls = botResponse.images.map((image) => image.url);
const validPosts = Promise.all(
posts
const validPosts = posts
.filter((post) => !imagesUrls.some((url) => url === post.url))
.map((post): Image => {
return { url: post.url, tags: post.tags };
})
);
});
logger.info(`Got ${validPosts.length} valid images from remote`);
if (validPosts.length > 0) {
return validPosts;
}
}
}