fedi-image-bot
/
fe-middleware
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented incremental pagination when no images are valid
Unit Tests with docker compose / unit-test (push) Successful in 19s Details
Build image / build (push) Successful in 43s Details

This commit is contained in:
Sugui 2024-06-15 20:50:22 +02:00
parent dd8066207e
commit 588970e18c
2 changed files with 28 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/services/gelbooruApiService.ts
View File

@ -6,8 +6,9 @@ import GelbooruServiceResponse from "src/types/GelbooruServiceResponse";
export const LIMIT = env.GELBOORU_IMAGES_PER_REQUEST || 100; export const LIMIT = env.GELBOORU_IMAGES_PER_REQUEST || 100;
export const TAGS = encodeURIComponent(env.GELBOORU_TAGS || ""); export const TAGS = encodeURIComponent(env.GELBOORU_TAGS || "");
export async function get(): Promise<GelbooruServiceResponse> { export async function get(pid?: number): Promise<GelbooruServiceResponse> {
const url: string = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=${LIMIT}&json=1&tags=${TAGS}`; const pidQueryParam = pid ? `&pid=${pid}` : "";
const url: string = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=${LIMIT}&json=1&tags=${TAGS}${pidQueryParam}`;
const response: GelbooruApiResponse = (await fetch(url).then( const response: GelbooruApiResponse = (await fetch(url).then(
async (res) => { async (res) => {

44
src/services/imageService.ts
View File

@ -2,22 +2,21 @@ import GelbooruServiceResponse from "src/types/GelbooruServiceResponse";
import Image from "src/types/Image"; import Image from "src/types/Image";
import logger from "src/logger"; import logger from "src/logger";
import { Mutex } from "async-mutex"; import { Mutex } from "async-mutex";
import * as gelbooruApiService from 'src/services/gelbooruApiService'; import * as gelbooruApiService from "src/services/gelbooruApiService";
import * as botApiService from 'src/services/botApiService'; import * as botApiService from "src/services/botApiService";
const mutex: Mutex = new Mutex(); const mutex: Mutex = new Mutex();
const postsQueue: Image[] = []; const postsQueue: Image[] = [];
// We wrap the function into a Mutex because it's not thread-safe // We wrap the function into a Mutex because it's not thread-safe
export async function get(): Promise<Image> { export async function get(): Promise<Image> {
return await mutex.runExclusive(() => unsafeGet()) return await mutex.runExclusive(() => unsafeGet());
} }
async function unsafeGet(): Promise<Image> { async function unsafeGet(): Promise<Image> {
while (postsQueue.length === 0) { if (postsQueue.length === 0) {
const validPosts = await getNewValidImages(); const validPosts = await getNewImages();
validPosts.map(post => postsQueue.push(post)); validPosts.map((post) => postsQueue.push(post));
logger.info(`Got ${validPosts.length} images from remote`);
} }
return popImage(); return popImage();
} }
@ -25,27 +24,34 @@ async function unsafeGet(): Promise<Image> {
function popImage(): Image { function popImage(): Image {
const image = postsQueue.pop(); const image = postsQueue.pop();
if (image) { if (image) {
return image return image;
} else { } else {
throw Error("Can't pop from an empty list"); throw Error("Can't pop from an empty list");
} }
} }
async function getNewValidImages(): Promise<Image[]> { /**
const gelbooruResponse: GelbooruServiceResponse = * Returns an array of images that are not present in the bot database
await gelbooruApiService.get(); */
const posts = gelbooruResponse.posts; async function getNewImages(): Promise<Image[]> {
for (let pageId = 0; ; pageId++) {
const gelbooruResponse: GelbooruServiceResponse =
await gelbooruApiService.get(pageId);
const posts = gelbooruResponse.posts;
const botResponse = await botApiService.getAll(); const botResponse = await botApiService.getAll();
const imagesUrls = botResponse.images.map((image) => image.url); const imagesUrls = botResponse.images.map((image) => image.url);
const validPosts = Promise.all( const validPosts = posts
posts
.filter((post) => !imagesUrls.some((url) => url === post.url)) .filter((post) => !imagesUrls.some((url) => url === post.url))
.map((post): Image => { .map((post): Image => {
return { url: post.url, tags: post.tags }; return { url: post.url, tags: post.tags };
}) });
);
return validPosts; logger.info(`Got ${validPosts.length} valid images from remote`);
if (validPosts.length > 0) {
return validPosts;
}
}
} }