fedi-image-bot
/
bot-mastodon-image-uploader
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'release/v1.0.0'
Unit Tests with docker compose and cargo / unit-test (push) Successful in 1m24s Details
Build image / build (push) Successful in 36m29s Details

This commit is contained in:
Alie 2024-02-23 08:18:04 +01:00
parent fec0b0b0b8 0e2123e8c5
commit def6bee627
10 changed files with 1146 additions and 600 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@ -32,3 +32,4 @@ LICENSE
README.md
config.toml
mastodon-data.toml
*.yaml

22
.gitea/workflows/01_test.yaml Normal file
View File

@ -0,0 +1,22 @@
name: Unit Tests with docker compose and cargo
on: [push, pull_request]
jobs:
unit-test:
container:
image: docker:dind
volumes:
- /data/.cache/act:/data/.cache/act
- /var/lib/docker/image:/var/lib/docker/image
- /var/lib/docker/overlay2:/var/lib/docker/overlay2
steps:
- name: Starting docker daemon
run: docker-init -- dockerd --host=unix:///var/run/docker.sock &
- name: Installing necessary packages
run: apk add nodejs git curl bash
- name: Check out repository code
uses: actions/checkout@v3
- name: Get access token from secret
run: echo "${{ secrets.MASTODON_SECRET }}" > mastodon-data.toml
- name: Run tests on docker
run: docker compose down -v && docker compose run bot t

52
.gitea/workflows/02_build.yaml Normal file
View File

@ -0,0 +1,52 @@
name: Build image
on:
push:
branches:
- main
- build
tags:
- v*
jobs:
build:
container:
image: docker:dind
volumes:
- /data/.cache/act:/data/.cache/act
- /var/lib/docker/image:/var/lib/docker/image
- /var/lib/docker/overlay2:/var/lib/docker/overlay2
steps:
- name: Starting docker daemon
run: docker-init -- dockerd --host=unix:///var/run/docker.sock &
- name: Installing necessary packages
run: apk add nodejs git curl bash
- name: Checkout
uses: actions/checkout@v3
- name: Docker meta
id: meta
uses: https://github.com/docker/metadata-action@v4
with:
# list of Docker images to use as base name for tags
images: |
git.fai.st/fedi-image-bot/mastodon-image-uploader-bot
# generate Docker tags based on the following events/attributes
tags: |
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
type=ref,event=branch
type=semver,pattern={{raw}}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to fai.st docker registry
uses: docker/login-action@v2
with:
registry: git.fai.st
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_PASS }}
- name: Build and push
uses: docker/build-push-action@v4
with:
platforms: linux/amd64,linux/arm64
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

1349
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

4
Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "mastodon-image-uploader-bot"
version = "0.3.1"
version = "1.0.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -11,7 +11,7 @@ reqwest = "0.11.14"
serde = "1.0.171"
toml = "0.8.8"
log = "0.4.19"
stderrlog = "0.5.4"
stderrlog = "0.6.0"
async-std = "1.12.0"
[dependencies.mastodon-async]

4
Dockerfile
View File

@ -12,7 +12,7 @@ ARG APP_NAME=mastodon-image-uploader-bot
FROM rust:${RUST_VERSION}-slim-bullseye AS build
ARG APP_NAME
WORKDIR /app
RUN apt update && apt install pkg-config openssl libssl-dev -y
# Build the application.
# Leverage a cache mount to /usr/local/cargo/registry/
# for downloaded dependencies and a cache mount to /app/target/ for
@ -23,8 +23,6 @@ WORKDIR /app
RUN --mount=type=bind,source=src,target=src \
--mount=type=bind,source=Cargo.toml,target=Cargo.toml \
--mount=type=bind,source=Cargo.lock,target=Cargo.lock \
--mount=type=cache,target=/app/target/ \
--mount=type=cache,target=/usr/local/cargo/registry/ \
<<EOF
set -e
cargo build --locked --release

41
compose.yaml Normal file
View File

@ -0,0 +1,41 @@
version: "3"
services:
mongodb:
image: mongo:bionic
container_name: mongodb
ports:
- "27017:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: password
MONGO_INITDB_DATABASE: bot
volumes:
- ./init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro
- mongodb_data:/data/db
bot-api:
image: git.fai.st/fedi-image-bot/bot-api:v1.0.0
container_name: bot-api
ports:
- "8080:8080"
depends_on:
- mongodb
environment:
MONGODB_URI: "mongodb://mongodb:27017/bot"
MONGODB_USER: "root"
MONGODB_PASS: "password"
JWTSECRET: "cooljwtsecret"
bot:
image: rust
container_name: bot
working_dir: /app
entrypoint: ["cargo"]
depends_on:
- bot-api
volumes:
- ./:/app:rw
volumes:
mongodb_data:

7
config.toml
View File

@ -4,10 +4,13 @@ instance = "https://awoo.fai.st"
bio = "Bot who posts images of sleeping girls every 6 hours."
[files]
urls = "./urls.csv"
posted = "./posted.csv"
tempfile = "/tmp/botimage.png"
[backend]
url = "http://bot-api:8080"
app = "tester"
secret = "test"
[errors]
maintainers = "@Sugui@awoo.fai.st @MeDueleLaTeta@awoo.fai.st"
out_of_images = "me quedé sin chicas"

10
init-mongo.js Normal file
View File

@ -0,0 +1,10 @@
db = new Mongo().getDB("bot");
db.createCollection('authorizations');
db.authorizations.insert([
{
app: "tester",
secret: "test",
}
]);

256
src/main.rs
View File

@ -2,12 +2,28 @@ use mastodon_async::entities::visibility::Visibility;
use mastodon_async::helpers::{cli, toml as masto_toml};
use mastodon_async::prelude::*;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::io::{Cursor, Write};
use std::io::Cursor;
use std::process::exit;
use std::time::Duration;
#[derive(Debug, Serialize)]
#[derive(Deserialize)]
struct ImagesWrap {
images: Vec<Image>,
}
#[derive(Deserialize, Clone, Eq, PartialEq, Debug)]
struct Image {
url: String,
_id: String,
status: String,
}
#[derive(Serialize)]
struct ImageStatus {
status: String,
}
#[derive(Serialize)]
struct AccountUpdate {
note: String,
}
@ -16,28 +32,34 @@ const CONFIG_FILENAME: &str = "config.toml";
type DynResult<T> = Result<T, Box<dyn std::error::Error>>;
#[derive(Deserialize)]
#[derive(Deserialize, Serialize, Default)]
struct Config {
bot: Bot,
files: Files,
backend: Backend,
errors: Errors,
files: Files,
}
#[derive(Deserialize)]
#[derive(Deserialize, Serialize, Default)]
struct Files {
tempfile: String,
}
#[derive(Deserialize, Serialize, Default)]
struct Bot {
name: String,
instance: String,
bio: String,
}
#[derive(Deserialize)]
struct Files {
posted: String,
urls: String,
tempfile: String,
#[derive(Deserialize, Serialize, Default)]
struct Backend {
url: String,
app: String,
secret: String,
}
#[derive(Deserialize)]
#[derive(Deserialize, Serialize, Default)]
struct Errors {
maintainers: String,
out_of_images: String,
@ -56,11 +78,12 @@ async fn main() -> DynResult<()> {
let config: Config = get_config();
let mastodon = get_account(&config).await;
match get_next_url(&config) {
Ok(url) => match url {
Some(url) => {
match get_next_url(&config).await {
Ok(image) => match image {
Some(image) => {
let mut retry: u8 = 0;
while let Err(err) = post_image(&mastodon, &url, &config, Visibility::Unlisted).await {
while let Err(err) = post_image(&mastodon, &image.url, &config, Visibility::Unlisted).await
{
log::warn!("Cannot post image, retry: {}, {}", retry, err);
async_std::task::sleep(Duration::new(1, 0)).await;
retry += 1;
@ -70,7 +93,7 @@ async fn main() -> DynResult<()> {
exit(1);
}
}
set_url_as_posted(&config, &url)?;
set_url_as_posted(&config, &image).await?;
let mut retry: u8 = 0;
while let Err(err) = update_bio(&mastodon, &config).await {
log::warn!("Cannot update bio, retry: {}, {}", retry, err);
@ -129,11 +152,26 @@ fn get_config() -> Config {
Ok(config) => config,
Err(err) => {
log::error!("Config file parsing unsuccesful: {}", err);
generate_config().unwrap();
log::info!("Please check the working directory to find a new config file");
exit(1);
}
}
}
fn generate_config() -> DynResult<()> {
let config = Config::default();
let config = toml::to_string(&config)?;
std::fs::write(CONFIG_FILENAME, config)?;
Ok(())
}
/// Parses the given filename to a config struct
fn parse_config(filename: &str) -> DynResult<Config> {
let toml_file = std::fs::read_to_string(filename)?; //.expect("No config file, consider getting the original one and modifing it");
Ok(toml::from_str(&toml_file)?) //("Malformed config file, check the original one for reference")
}
async fn get_account(config: &Config) -> Mastodon {
if let Ok(data) = masto_toml::from_file("mastodon-data.toml") {
Mastodon::from(data)
@ -148,33 +186,67 @@ async fn get_account(config: &Config) -> Mastodon {
}
}
/// Parses the given filename to a config struct
fn parse_config(filename: &str) -> DynResult<Config> {
let toml_file = std::fs::read_to_string(filename)?; //.expect("No config file, consider getting the original one and modifing it");
Ok(toml::from_str(&toml_file)?) //("Malformed config file, check the original one for reference")
}
async fn get_next_url(config: &Config) -> DynResult<Option<Image>> {
let images: ImagesWrap = reqwest::get(format!(
"{}/images?status=available&limit=1",
config.backend.url
))
.await?
.json()
.await?;
fn get_next_url(config: &Config) -> DynResult<Option<String>> {
let binding = std::fs::read_to_string(&config.files.posted)?; //.expect("Posted file not found");
let posted = binding.lines().collect::<HashSet<&str>>();
let binding = std::fs::read_to_string(&config.files.urls)?; //.expect("Urls file not found");
let urls = binding.lines().collect::<HashSet<&str>>();
let urls = urls.difference(&posted).collect::<Vec<_>>();
if urls.is_empty() {
let images = images.images;
if images.is_empty() {
Ok(None)
} else {
Ok(Some(urls[0].to_string().clone()))
Ok(Some(images[0].clone()))
}
}
fn set_url_as_posted(config: &Config, url: &String) -> DynResult<()> {
let mut file = std::fs::OpenOptions::new()
.write(true)
.append(true) // This is needed to append to file
.open(&config.files.posted)?; //.expect("Cannot open posted file"); // Maybe we should retry just in case
writeln!(file, "{}", url)?; //.expect("Cannot write to posted file"); // maybe we should retry tbh
log::info!("Set url {} as posted", url);
Ok(())
#[derive(Serialize)]
struct Auth {
app: String,
secret: String,
}
#[derive(Deserialize)]
struct Token {
token: String,
}
async fn set_url_as_posted(config: &Config, image: &Image) -> DynResult<()> {
let client = reqwest::Client::new();
let auth = &Auth {
app: config.backend.app.to_string(),
secret: config.backend.secret.to_string(),
};
let token: Token = client
.post(&format!("{}/login", config.backend.url))
.json(auth)
.send()
.await?
.json()
.await?;
let response = client
.put(&format!("{}/images/{}", config.backend.url, image._id))
.bearer_auth(token.token)
.json(&ImageStatus {
status: "consumed".to_string(),
})
.send()
.await?;
if response.status().is_success() {
log::info!("Set url {} as posted", image.url);
Ok(())
} else {
let error = response.text().await?;
log::error!("Set url response: {}", error);
Err(error.into())
}
}
async fn post_image(
@ -201,12 +273,12 @@ async fn post_image(
}
async fn update_bio(account: &Mastodon, config: &Config) -> DynResult<()> {
let binding = std::fs::read_to_string(&config.files.posted)?; //.expect("Posted file not found");
let posted = binding.lines().collect::<HashSet<&str>>();
let binding = std::fs::read_to_string(&config.files.urls)?; //.expect("Url file not found");
let urls = binding.lines().collect::<HashSet<&str>>();
let images: ImagesWrap = reqwest::get(format!("{}/images?status=available", config.backend.url))
.await?
.json()
.await?;
let remaining = urls.difference(&posted).count();
let remaining = images.images.len();
let client = reqwest::Client::new();
let account_update = AccountUpdate {
@ -323,35 +395,93 @@ mod tests {
assert_eq!(response.status(), StatusCode::OK)
}
#[test]
fn set_as_posted_works() {
#[tokio::test]
async fn set_as_posted_works() {
let config = get_config();
const IMAGE: &str = "https://picsum.photos/id/2";
let expected = insert_image(&config, IMAGE).await.unwrap();
std::fs::write(&config.files.posted, "").unwrap();
set_url_as_posted(&config, &TEST_URL.to_string()).unwrap();
let file = std::fs::read_to_string(&config.files.posted).unwrap();
let url = file.lines().next_back().unwrap();
assert_eq!(url, TEST_URL)
set_url_as_posted(&config, &expected).await.unwrap();
let image: ImageWrap = reqwest::get(format!("{}/images/{}", config.backend.url, expected._id))
.await
.unwrap()
.json()
.await
.unwrap();
let image = image.image;
assert_eq!(image.status, "consumed");
}
#[test]
fn get_next_url_works() {
#[tokio::test]
async fn get_next_url_works() {
let config = get_config();
// Reset file
std::fs::write(&config.files.posted, "").unwrap();
const IMAGE: &str = "https://picsum.photos/id/1";
let expected = insert_image(&config, IMAGE).await.unwrap();
// Get test url
let url = get_next_url(&config).unwrap().unwrap();
assert_eq!(url, TEST_URL);
let image = get_next_url(&config).await.unwrap().unwrap();
assert_eq!(image.url, IMAGE);
set_url_as_posted(&config, &TEST_URL.to_string()).unwrap();
let file = std::fs::read_to_string(&config.files.posted).unwrap();
let url = file.lines().next_back().unwrap();
assert_eq!(url, TEST_URL);
set_url_as_posted(&config, &expected).await.unwrap();
let image: ImageWrap = reqwest::get(format!("{}/images/{}", config.backend.url, expected._id))
.await
.unwrap()
.json()
.await
.unwrap();
let image = image.image;
assert_eq!(image.status, "consumed");
// Test that now it does not get it
let url = get_next_url(&config).unwrap();
assert_eq!(url, None);
let image = get_next_url(&config).await.unwrap();
assert_eq!(image, None);
}
async fn insert_image(config: &Config, url: &str) -> DynResult<Image> {
let client = reqwest::Client::new();
let auth = &Auth {
app: config.backend.app.to_string(),
secret: config.backend.secret.to_string(),
};
let token: Token = client
.post(&format!("{}/login", config.backend.url))
.json(auth)
.send()
.await?
.json()
.await?;
let response = client
.post(&format!("{}/images", config.backend.url))
.bearer_auth(&token.token)
.json(&Insert {
url: url.to_string(),
status: "available".to_string(),
tags: vec![],
})
.send()
.await?;
if response.status().is_success() {
let response: ImageWrap = response.json().await?;
return Ok(response.image);
}
dbg!(response.text().await?);
panic!();
}
#[derive(Serialize)]
struct Insert {
url: String,
status: String,
tags: Vec<String>,
}
#[derive(Deserialize)]
struct ImageWrap {
image: Image,
}
}