Merge branch 'release/v1.0.0'

.dockerignore

@@ -32,3 +32,4 @@ LICENSE
 README.md
 config.toml
 mastodon-data.toml
+*.yaml

.gitea/workflows/01_test.yaml

@@ -0,0 +1,22 @@
+name: Unit Tests with docker compose and cargo
+on: [push, pull_request]
+
+jobs:
+  unit-test:
+    container:
+      image: docker:dind
+      volumes:
+        - /data/.cache/act:/data/.cache/act
+        - /var/lib/docker/image:/var/lib/docker/image
+        - /var/lib/docker/overlay2:/var/lib/docker/overlay2
+    steps:
+      - name: Starting docker daemon
+        run: docker-init -- dockerd --host=unix:///var/run/docker.sock &
+      - name: Installing necessary packages
+        run: apk add nodejs git curl bash
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Get access token from secret
+        run: echo "${{ secrets.MASTODON_SECRET }}" > mastodon-data.toml
+      - name: Run tests on docker
+        run: docker compose down -v && docker compose run bot t

.gitea/workflows/02_build.yaml

@@ -0,0 +1,52 @@
+name: Build image
+on:
+  push:
+    branches:
+      - main
+      - build
+    tags:
+      - v*
+      
+jobs:
+  build:
+    container:
+      image: docker:dind
+      volumes:
+        - /data/.cache/act:/data/.cache/act
+        - /var/lib/docker/image:/var/lib/docker/image
+        - /var/lib/docker/overlay2:/var/lib/docker/overlay2
+    steps:
+      - name: Starting docker daemon
+        run: docker-init -- dockerd --host=unix:///var/run/docker.sock &
+      - name: Installing necessary packages
+        run: apk add nodejs git curl bash
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Docker meta
+        id: meta
+        uses: https://github.com/docker/metadata-action@v4
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            git.fai.st/fedi-image-bot/mastodon-image-uploader-bot
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+            type=ref,event=branch
+            type=semver,pattern={{raw}}            
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to fai.st docker registry
+        uses: docker/login-action@v2
+        with:
+          registry: git.fai.st
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASS }}
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          platforms: linux/amd64,linux/arm64
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mastodon-image-uploader-bot"
-version = "0.3.1"
+version = "1.0.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -11,7 +11,7 @@ reqwest = "0.11.14"
 serde = "1.0.171"
 toml = "0.8.8"
 log = "0.4.19"
-stderrlog = "0.5.4"
+stderrlog = "0.6.0"
 async-std = "1.12.0"
 
 [dependencies.mastodon-async]

Dockerfile

@@ -12,7 +12,7 @@ ARG APP_NAME=mastodon-image-uploader-bot
 FROM rust:${RUST_VERSION}-slim-bullseye AS build
 ARG APP_NAME
 WORKDIR /app
-
+RUN apt update && apt install pkg-config openssl libssl-dev -y
 # Build the application.
 # Leverage a cache mount to /usr/local/cargo/registry/
 # for downloaded dependencies and a cache mount to /app/target/ for
@@ -23,8 +23,6 @@ WORKDIR /app
 RUN --mount=type=bind,source=src,target=src \
     --mount=type=bind,source=Cargo.toml,target=Cargo.toml \
     --mount=type=bind,source=Cargo.lock,target=Cargo.lock \
-    --mount=type=cache,target=/app/target/ \
-    --mount=type=cache,target=/usr/local/cargo/registry/ \
     <<EOF
 set -e
 cargo build --locked --release

compose.yaml

@@ -0,0 +1,41 @@
+version: "3"
+
+services:
+  mongodb:
+    image: mongo:bionic
+    container_name: mongodb
+    ports:
+      - "27017:27017"
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: password
+      MONGO_INITDB_DATABASE: bot
+    volumes:
+      - ./init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro
+      - mongodb_data:/data/db
+
+  bot-api:
+    image: git.fai.st/fedi-image-bot/bot-api:v1.0.0
+    container_name: bot-api
+    ports:
+      - "8080:8080"
+    depends_on:
+      - mongodb
+    environment:
+      MONGODB_URI: "mongodb://mongodb:27017/bot"
+      MONGODB_USER: "root"
+      MONGODB_PASS: "password"
+      JWTSECRET: "cooljwtsecret"
+
+  bot:
+    image: rust
+    container_name: bot
+    working_dir: /app
+    entrypoint: ["cargo"]
+    depends_on:
+      - bot-api
+    volumes:
+      - ./:/app:rw
+
+volumes:
+  mongodb_data:

config.toml

@@ -4,10 +4,13 @@ instance = "https://awoo.fai.st"
 bio = "Bot who posts images of sleeping girls every 6 hours."
 
 [files]
-urls = "./urls.csv"
-posted = "./posted.csv"
 tempfile = "/tmp/botimage.png"
 
+[backend]
+url = "http://bot-api:8080"
+app = "tester"
+secret = "test"
+
 [errors]
 maintainers = "@Sugui@awoo.fai.st @MeDueleLaTeta@awoo.fai.st"
 out_of_images = "me quedé sin chicas"

init-mongo.js

@@ -0,0 +1,10 @@
+db = new Mongo().getDB("bot");
+
+db.createCollection('authorizations');
+
+db.authorizations.insert([
+	{
+    app: "tester",
+    secret: "test",
+  }
+]);

src/main.rs

@@ -2,12 +2,28 @@ use mastodon_async::entities::visibility::Visibility;
 use mastodon_async::helpers::{cli, toml as masto_toml};
 use mastodon_async::prelude::*;
 use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
+use std::io::Cursor;
-use std::io::{Cursor, Write};
 use std::process::exit;
 use std::time::Duration;
 
-#[derive(Debug, Serialize)]
+#[derive(Deserialize)]
+struct ImagesWrap {
+  images: Vec<Image>,
+}
+
+#[derive(Deserialize, Clone, Eq, PartialEq, Debug)]
+struct Image {
+  url: String,
+  _id: String,
+  status: String,
+}
+
+#[derive(Serialize)]
+struct ImageStatus {
+  status: String,
+}
+
+#[derive(Serialize)]
 struct AccountUpdate {
   note: String,
 }
@@ -16,28 +32,34 @@ const CONFIG_FILENAME: &str = "config.toml";
 
 type DynResult<T> = Result<T, Box<dyn std::error::Error>>;
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize, Default)]
 struct Config {
   bot: Bot,
-  files: Files,
+  backend: Backend,
   errors: Errors,
+  files: Files,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize, Default)]
+struct Files {
+  tempfile: String,
+}
+
+#[derive(Deserialize, Serialize, Default)]
 struct Bot {
   name: String,
   instance: String,
   bio: String,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize, Default)]
-struct Files {
+struct Backend {
-  posted: String,
+  url: String,
-  urls: String,
+  app: String,
-  tempfile: String,
+  secret: String,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize, Default)]
 struct Errors {
   maintainers: String,
   out_of_images: String,
@@ -56,11 +78,12 @@ async fn main() -> DynResult<()> {
   let config: Config = get_config();
   let mastodon = get_account(&config).await;
 
-  match get_next_url(&config) {
+  match get_next_url(&config).await {
-    Ok(url) => match url {
+    Ok(image) => match image {
-      Some(url) => {
+      Some(image) => {
         let mut retry: u8 = 0;
-        while let Err(err) = post_image(&mastodon, &url, &config, Visibility::Unlisted).await {
+        while let Err(err) = post_image(&mastodon, &image.url, &config, Visibility::Unlisted).await
+        {
           log::warn!("Cannot post image, retry: {}, {}", retry, err);
           async_std::task::sleep(Duration::new(1, 0)).await;
           retry += 1;
@@ -70,7 +93,7 @@ async fn main() -> DynResult<()> {
             exit(1);
           }
         }
-        set_url_as_posted(&config, &url)?;
+        set_url_as_posted(&config, &image).await?;
         let mut retry: u8 = 0;
         while let Err(err) = update_bio(&mastodon, &config).await {
           log::warn!("Cannot update bio, retry: {}, {}", retry, err);
@@ -129,11 +152,26 @@ fn get_config() -> Config {
     Ok(config) => config,
     Err(err) => {
       log::error!("Config file parsing unsuccesful: {}", err);
+      generate_config().unwrap();
+      log::info!("Please check the working directory to find a new config file");
       exit(1);
     }
   }
 }
 
+fn generate_config() -> DynResult<()> {
+  let config = Config::default();
+  let config = toml::to_string(&config)?;
+  std::fs::write(CONFIG_FILENAME, config)?;
+  Ok(())
+}
+
+/// Parses the given filename to a config struct
+fn parse_config(filename: &str) -> DynResult<Config> {
+  let toml_file = std::fs::read_to_string(filename)?; //.expect("No config file, consider getting the original one and modifing it");
+  Ok(toml::from_str(&toml_file)?) //("Malformed config file, check the original one for reference")
+}
+
 async fn get_account(config: &Config) -> Mastodon {
   if let Ok(data) = masto_toml::from_file("mastodon-data.toml") {
     Mastodon::from(data)
@@ -148,33 +186,67 @@ async fn get_account(config: &Config) -> Mastodon {
   }
 }
 
-/// Parses the given filename to a config struct
+async fn get_next_url(config: &Config) -> DynResult<Option<Image>> {
-fn parse_config(filename: &str) -> DynResult<Config> {
+  let images: ImagesWrap = reqwest::get(format!(
-  let toml_file = std::fs::read_to_string(filename)?; //.expect("No config file, consider getting the original one and modifing it");
+    "{}/images?status=available&limit=1",
-  Ok(toml::from_str(&toml_file)?) //("Malformed config file, check the original one for reference")
+    config.backend.url
-}
+  ))
+  .await?
+  .json()
+  .await?;
 
-fn get_next_url(config: &Config) -> DynResult<Option<String>> {
+  let images = images.images;
-  let binding = std::fs::read_to_string(&config.files.posted)?; //.expect("Posted file not found");
+
-  let posted = binding.lines().collect::<HashSet<&str>>();
+  if images.is_empty() {
-  let binding = std::fs::read_to_string(&config.files.urls)?; //.expect("Urls file not found");
-  let urls = binding.lines().collect::<HashSet<&str>>();
-  let urls = urls.difference(&posted).collect::<Vec<_>>();
-  if urls.is_empty() {
     Ok(None)
   } else {
-    Ok(Some(urls[0].to_string().clone()))
+    Ok(Some(images[0].clone()))
   }
 }
 
-fn set_url_as_posted(config: &Config, url: &String) -> DynResult<()> {
+#[derive(Serialize)]
-  let mut file = std::fs::OpenOptions::new()
+struct Auth {
-    .write(true)
+  app: String,
-    .append(true) // This is needed to append to file
+  secret: String,
-    .open(&config.files.posted)?; //.expect("Cannot open posted file"); // Maybe we should retry just in case
+}
-  writeln!(file, "{}", url)?; //.expect("Cannot write to posted file"); // maybe we should retry tbh
+
-  log::info!("Set url {} as posted", url);
+#[derive(Deserialize)]
+struct Token {
+  token: String,
+}
+
+async fn set_url_as_posted(config: &Config, image: &Image) -> DynResult<()> {
+  let client = reqwest::Client::new();
+
+  let auth = &Auth {
+    app: config.backend.app.to_string(),
+    secret: config.backend.secret.to_string(),
+  };
+
+  let token: Token = client
+    .post(&format!("{}/login", config.backend.url))
+    .json(auth)
+    .send()
+    .await?
+    .json()
+    .await?;
+
+  let response = client
+    .put(&format!("{}/images/{}", config.backend.url, image._id))
+    .bearer_auth(token.token)
+    .json(&ImageStatus {
+      status: "consumed".to_string(),
+    })
+    .send()
+    .await?;
+  if response.status().is_success() {
+    log::info!("Set url {} as posted", image.url);
     Ok(())
+  } else {
+    let error = response.text().await?;
+    log::error!("Set url response: {}", error);
+    Err(error.into())
+  }
 }
 
 async fn post_image(
@@ -201,12 +273,12 @@ async fn post_image(
 }
 
 async fn update_bio(account: &Mastodon, config: &Config) -> DynResult<()> {
-  let binding = std::fs::read_to_string(&config.files.posted)?; //.expect("Posted file not found");
+  let images: ImagesWrap = reqwest::get(format!("{}/images?status=available", config.backend.url))
-  let posted = binding.lines().collect::<HashSet<&str>>();
+    .await?
-  let binding = std::fs::read_to_string(&config.files.urls)?; //.expect("Url file not found");
+    .json()
-  let urls = binding.lines().collect::<HashSet<&str>>();
+    .await?;
 
-  let remaining = urls.difference(&posted).count();
+  let remaining = images.images.len();
   let client = reqwest::Client::new();
 
   let account_update = AccountUpdate {
@@ -323,35 +395,93 @@ mod tests {
     assert_eq!(response.status(), StatusCode::OK)
   }
 
-  #[test]
+  #[tokio::test]
-  fn set_as_posted_works() {
+  async fn set_as_posted_works() {
     let config = get_config();
+    const IMAGE: &str = "https://picsum.photos/id/2";
+    let expected = insert_image(&config, IMAGE).await.unwrap();
 
-    std::fs::write(&config.files.posted, "").unwrap();
+    set_url_as_posted(&config, &expected).await.unwrap();
-
+    let image: ImageWrap = reqwest::get(format!("{}/images/{}", config.backend.url, expected._id))
-    set_url_as_posted(&config, &TEST_URL.to_string()).unwrap();
+      .await
-    let file = std::fs::read_to_string(&config.files.posted).unwrap();
+      .unwrap()
-    let url = file.lines().next_back().unwrap();
+      .json()
-    assert_eq!(url, TEST_URL)
+      .await
+      .unwrap();
+    let image = image.image;
+    assert_eq!(image.status, "consumed");
   }
 
-  #[test]
+  #[tokio::test]
-  fn get_next_url_works() {
+  async fn get_next_url_works() {
     let config = get_config();
-    // Reset file
+
-    std::fs::write(&config.files.posted, "").unwrap();
+    const IMAGE: &str = "https://picsum.photos/id/1";
+    let expected = insert_image(&config, IMAGE).await.unwrap();
 
     // Get test url
-    let url = get_next_url(&config).unwrap().unwrap();
+    let image = get_next_url(&config).await.unwrap().unwrap();
-    assert_eq!(url, TEST_URL);
+    assert_eq!(image.url, IMAGE);
 
-    set_url_as_posted(&config, &TEST_URL.to_string()).unwrap();
+    set_url_as_posted(&config, &expected).await.unwrap();
-    let file = std::fs::read_to_string(&config.files.posted).unwrap();
+    let image: ImageWrap = reqwest::get(format!("{}/images/{}", config.backend.url, expected._id))
-    let url = file.lines().next_back().unwrap();
+      .await
-    assert_eq!(url, TEST_URL);
+      .unwrap()
+      .json()
+      .await
+      .unwrap();
+    let image = image.image;
+    assert_eq!(image.status, "consumed");
 
     // Test that now it does not get it
-    let url = get_next_url(&config).unwrap();
+    let image = get_next_url(&config).await.unwrap();
-    assert_eq!(url, None);
+    assert_eq!(image, None);
+  }
+
+  async fn insert_image(config: &Config, url: &str) -> DynResult<Image> {
+    let client = reqwest::Client::new();
+
+    let auth = &Auth {
+      app: config.backend.app.to_string(),
+      secret: config.backend.secret.to_string(),
+    };
+
+    let token: Token = client
+      .post(&format!("{}/login", config.backend.url))
+      .json(auth)
+      .send()
+      .await?
+      .json()
+      .await?;
+
+    let response = client
+      .post(&format!("{}/images", config.backend.url))
+      .bearer_auth(&token.token)
+      .json(&Insert {
+        url: url.to_string(),
+        status: "available".to_string(),
+        tags: vec![],
+      })
+      .send()
+      .await?;
+
+    if response.status().is_success() {
+      let response: ImageWrap = response.json().await?;
+      return Ok(response.image);
+    }
+    dbg!(response.text().await?);
+    panic!();
+  }
+
+  #[derive(Serialize)]
+  struct Insert {
+    url: String,
+    status: String,
+    tags: Vec<String>,
+  }
+
+  #[derive(Deserialize)]
+  struct ImageWrap {
+    image: Image,
   }
 }