Implemented incremental pagination when no images are valid

src/services/gelbooruApiService.ts

@@ -6,8 +6,9 @@ import GelbooruServiceResponse from "src/types/GelbooruServiceResponse";
 export const LIMIT = env.GELBOORU_IMAGES_PER_REQUEST || 100;
 export const TAGS = encodeURIComponent(env.GELBOORU_TAGS || "");
 
-export async function get(): Promise<GelbooruServiceResponse> {
+export async function get(pid?: number): Promise<GelbooruServiceResponse> {
-  const url: string = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=${LIMIT}&json=1&tags=${TAGS}`;
+  const pidQueryParam = pid ? `&pid=${pid}` : "";
+  const url: string = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=${LIMIT}&json=1&tags=${TAGS}${pidQueryParam}`;
 
   const response: GelbooruApiResponse = (await fetch(url).then(
     async (res) => {

src/services/imageService.ts

@@ -2,22 +2,21 @@ import GelbooruServiceResponse from "src/types/GelbooruServiceResponse";
 import Image from "src/types/Image";
 import logger from "src/logger";
 import { Mutex } from "async-mutex";
-import * as gelbooruApiService from 'src/services/gelbooruApiService';
+import * as gelbooruApiService from "src/services/gelbooruApiService";
-import * as botApiService from 'src/services/botApiService';
+import * as botApiService from "src/services/botApiService";
 
 const mutex: Mutex = new Mutex();
 const postsQueue: Image[] = [];
 
 // We wrap the function into a Mutex because it's not thread-safe
 export async function get(): Promise<Image> {
-  return await mutex.runExclusive(() => unsafeGet())
+  return await mutex.runExclusive(() => unsafeGet());
 }
 
 async function unsafeGet(): Promise<Image> {
-  while (postsQueue.length === 0) {
+  if (postsQueue.length === 0) {
-    const validPosts = await getNewValidImages();
+    const validPosts = await getNewImages();
-    validPosts.map(post => postsQueue.push(post));
+    validPosts.map((post) => postsQueue.push(post));
-    logger.info(`Got ${validPosts.length} images from remote`);
   }
   return popImage();
 }
@@ -25,27 +24,34 @@ async function unsafeGet(): Promise<Image> {
 function popImage(): Image {
   const image = postsQueue.pop();
   if (image) {
-    return image
+    return image;
   } else {
     throw Error("Can't pop from an empty list");
   }
 }
 
-async function getNewValidImages(): Promise<Image[]> {
+/**
+ * Returns an array of images that are not present in the bot database
+ */
+async function getNewImages(): Promise<Image[]> {
+  for (let pageId = 0; ; pageId++) {
     const gelbooruResponse: GelbooruServiceResponse =
-    await gelbooruApiService.get();
+      await gelbooruApiService.get(pageId);
     const posts = gelbooruResponse.posts;
 
     const botResponse = await botApiService.getAll();
     const imagesUrls = botResponse.images.map((image) => image.url);
 
-  const validPosts = Promise.all(
+    const validPosts = posts
-    posts
       .filter((post) => !imagesUrls.some((url) => url === post.url))
       .map((post): Image => {
         return { url: post.url, tags: post.tags };
-      })
+      });
-  );
 
+    logger.info(`Got ${validPosts.length} valid images from remote`);
+
+    if (validPosts.length > 0) {
       return validPosts;
+    }
+  }
 }