feat: Implement chinese sentence parsing & production creation

data/ignored.list

@@ -1,3 +1,14 @@
+吲
+Seems to only be used in the name of a chemical to make the sound "in"...
+Wtf, kinda useless
+
+凢
+Just a variant on 凡, doesn't seem to be of much value.
+
+凼
+Can't see any uses, it would be easy to add but it's literally not used.
+Revisit if this isn't the case.
+
 刄
 Just an 異体字 of 刃... Dunno, doesn't make much sense to me
 

data/lists/ebook.list

@@ -193,6 +193,6 @@
 姱裼蟫蚜炅佟綦虁剕甗絺塤綌璦菹貙詡偀蝪篶覬覦餤踹摔鷖躺腁霪睟摽焄鷆听哎珱昰鯳魵鮴
 鞁莇筴黻倢伃栝蕡䗪惷蔛蕢鱉尃豨蓚塋郅堋竓纊騭栫祅喼抳犾罧揵檑餺飥邙刕醮羡愡敕兹滚
 瘕軺蔲汛幗紱嘽膄崫砡槝糫薏鵟癭菔怔忡㾮鉀梍凞鈺蓪枘掫匾傕卬臧轀沚椴嵒轣噐搞您櫤誐
-儗撇儵呕槫鳫裵澑猬杔膛鶴騸菇琇鐲靖杦畆鰩划呿秂瞪弸蕫皤唏凬帮摒棙岼湶砿飃拕泆瀅氂
-鬭斁懟阨頥忼擕隳熛鄄渮濰紓蜇阴竽傈轌蕋駉盻禘鄹愀纆絏杇棖鍇騂魋忞葸悾勉忮踧踖紾飪
-阼喭僎訒慝柙櫝肸訕耰蓧鼗璩靛坷煮糲
+儗撇儵呕槫鳫裵澑猬杔膛鶴騸菇琇鐲靖杦畆鰩划呿秂瞪弸蕫皤唏凬帮摒棙岼湶砿飃枴蠧矚彑
+嫻拕泆瀅氂鬭斁懟阨頥忼擕隳熛鄄渮濰紓蜇阴竽傈轌蕋駉盻禘鄹愀纆絏杇棖鍇騂魋忞葸悾勉
+忮踧踖紾飪阼喭僎訒慝柙櫝肸訕耰蓧鼗璩靛坷煮糲

src/Command/CreateChineseProductionCommand.php

@@ -0,0 +1,73 @@
+<?php
+
+namespace App\Command;
+
+use App\Entity\ChineseProductionNote;
+use App\Entity\ChineseSentenceNote;
+use App\Entity\SentenceNote;
+use App\Service\AnkiService;
+
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+#[AsCommand('app:create:chinese:production', 'Create new listening Anki Cards')]
+class CreateChineseProductionCommand extends Command
+{
+    public function __construct(
+        private AnkiService $ankiService,
+    ) {
+        parent::__construct();
+    }
+
+    protected function configure(): void
+    {
+        //$this->addArgument(
+        //    'count',
+        //    InputArgument::REQUIRED,
+        //    'Amount of cards to make',
+        //);
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $csns = $this->ankiService->getAllFromClass(ChineseSentenceNote::class);
+        $cpns = $this->ankiService->getAllFromClass(ChineseProductionNote::class);
+
+        $existentTerms = [];
+        foreach ($cpns as $productionNote) {
+            $existentTerms[$productionNote->getTerm()->getKanji()] = null;
+        }
+
+        ////$newNotesCount = intval($input->getArgument('count'));
+
+        $newProductionNotes = [];
+        foreach ($csns as $sentenceNote) {
+            foreach ($sentenceNote->getTerms() as $term) {
+                $termStr = $term->getKanji();
+
+                if (key_exists($termStr, $existentTerms)) continue;
+
+                $existentTerms[$term->getKanji()] = null;
+                $newProductionNotes[] = ChineseProductionNote::fromNote($sentenceNote, $term);
+                //if (count($newProductionNotes) >= $newNotesCount) break 2;
+            }
+        }
+
+        foreach ($newProductionNotes as $newNote) {
+            $this->ankiService->addNote($newNote);
+        }
+
+        printf(
+            <<<FMNT
+            max usage:     %0.2f MiB
+            current usage: %0.2f MiB\n
+            FMNT,
+            memory_get_peak_usage() / 1024 / 1024,
+            memory_get_usage() / 1024 / 1024,
+        );
+
+        return Command::SUCCESS;
+    }
+}

src/Controller/KanjiController.php

@@ -9,6 +9,7 @@ use App\Utils\Japanese;
 use App\Utils\Number;
 
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+use Symfony\Component\HttpClient\Exception\TransportException;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\HttpKernel\Attribute\MapQueryParameter;
@@ -176,7 +177,11 @@ class KanjiController extends AbstractController
             "{$this->getParameter('kernel.project_dir')}/data/kanken-links.json",
         ), true);
 
+        $terms = null;
+        try {
             $terms = $this->anki->searchTerms($charStr);
+        } catch (TransportException) {
+        }
 
         $ebookRef = require "$this->varBasepath/ebook-ref.php";
 

src/Entity/ChineseProductionNote.php

@@ -0,0 +1,90 @@
+<?php
+
+namespace App\Entity;
+
+use App\Utils\Japanese;
+
+class ChineseProductionNote extends Note
+{
+    const MODEL_NAME = 'Chinese production';
+    const DECK       = '汉语::汉字';
+
+    private ?array $mediaInfo = [];
+    private ?Term $term = null;
+
+    public static function fromNote(Note $origNote, Term $term): self
+    {
+        $slNote = new self();
+        foreach (get_object_vars($origNote) as $prop => $value) {
+            $slNote->$prop = $value;
+        }
+
+        // Related fields are updated using the setter
+        $slNote->setTerm($term);
+        // Reset relations and basic data
+        $slNote->id      = null;
+        $slNote->model   = self::MODEL_NAME;
+        $slNote->cardIds = [];
+
+        return $slNote;
+    }
+
+    // -------------------------------------------------- Getters & setters ---
+
+    public function getTerm(): Term
+    {
+        return $this->term;
+    }
+    public function setTerm(Term $term): static
+    {
+        $hanzi = Japanese::getKanjiList($term->getKanji());
+        $pinyin = explode(' ', $term->getReading());
+
+        foreach ($pinyin as $key => $iPinyin) {
+            $color = ChineseTone::fromPinyin($iPinyin)->getColor();
+            $spanTmpl = '<span style="color: %s;">%s</span>';
+
+            $hanzi[$key]  = sprintf($spanTmpl, $color, $hanzi[$key]);
+            $pinyin[$key] = sprintf($spanTmpl, $color, $iPinyin);
+        }
+
+        if (!isset($term->audio)) dd($term);
+        $this->fields['VocabHanzi']  = implode('', $hanzi);
+        $this->fields['VocabPinyin'] = implode(' ', $pinyin);
+        $this->fields['VocabDef']    = $term->toAnkiVocabDef();
+        $this->fields['VocabAudio']  = $term->audio;
+        $this->fields['SentHanzi']   = Note::stringHighlight(
+            $this->fields['SentHanzi'],
+            $term->getKanji(),
+        );
+        $this->term = $term;
+        return $this;
+    }
+
+
+    // ------------------------------------------------------- Anki-related ---
+
+    /** @param array<string, string> $noteInfo */
+    public static function fromAnki(array $noteInfo): static
+    {
+        $note = parent::fromAnki($noteInfo);
+
+        if ($note->getModel() !== self::MODEL_NAME) {
+            throw new \Exception('Trying to parse wrong model');
+        }
+
+        $note->mediaInfo = Note::parseMediaInfo($note->fields['Notes']);
+
+        // Set VocabKanji field
+        $note->term = Term::fromNoteFields($note->fields)[0] ?? null;
+        if ($note->term === null) {
+            throw new \Exception("Couldn't get term for Listening card");
+        }
+
+        return $note;
+    }
+
+
+    // ---------------------------------------------------- Derived methods ---
+
+}

src/Entity/ChineseSentenceNote.php

@@ -0,0 +1,172 @@
+<?php
+
+namespace App\Entity;
+
+use App\Utils\Japanese;
+
+class ChineseSentenceNote extends Note
+{
+    const MODEL_NAME = 'Chinese sentences';
+    const DECK       = '汉语::朗读';
+
+    private ?array $mediaInfo = [];
+    /** @var list<Term> */
+    private array  $terms = [];
+
+    // -------------------------------------------------- Getters & setters ---
+
+    /** @return list<Term> */
+    public function getTerms(): array
+    {
+        return $this->terms;
+    }
+    /** @param list<Term> $terms */
+    public function setTerms(array $terms): static
+    {
+        $this->terms = $terms;
+        return $this;
+    }
+
+
+    // ------------------------------------------------------- Anki-related ---
+
+    /** @param array<string, string> $noteInfo */
+    public static function fromAnki(array $noteInfo): static
+    {
+        $note = parent::fromAnki($noteInfo);
+
+        if ($note->getModel() !== self::MODEL_NAME) {
+            throw new \Exception('Trying to parse wrong model');
+        }
+
+        $note->mediaInfo = Note::parseMediaInfo($note->fields['Notes']);
+
+        // Set VocabKanji field
+        $note->terms = Term::fromNoteFields($note->fields);
+
+        // If unable to, create them from the highlighted parts in the sentence
+        if (empty($note->terms)) {
+            foreach ($note->getHighlightedKanji() as $highlighedKanji) {
+                $term = new Term();
+                $term->kanji = $highlighedKanji;
+                $term->definitionEn = null;
+                $term->definitionJp = null;
+                $note->terms[] = $term;
+            }
+        }
+
+        // Set to null whatever is null
+        $readings = array_map(
+            fn($x) => in_array($x, ['_', '＿', '']) ? null : $x,
+            explode('｜', $note->fields['VocabPinyin']),
+        );
+
+        if (count($readings) !== count($note->terms)) throw new \Exception(sprintf(
+            <<<FMT
+            Number of terms and readings is not equal.
+            Expected %d readings, got %d.
+
+            Hanzi:    "%s"
+            Readings: "%s",
+            FMT,
+            count($note->terms),
+            count($readings),
+            implode('|', array_map(fn(Term $x) => $x->kanji, $note->terms)),
+            implode('|', $readings),
+        ));
+
+
+        // Set readings from furigana field
+        foreach ($note->terms as $key => &$term) {
+            // already has a reading
+            if (null !== $term->getReading()) continue;
+            // reading not specified
+            $reading = $readings[$key] ?? null;
+            if (null === $reading) continue;
+
+            $reading = explode(' ', $reading);
+            if (count($reading) !== mb_strlen($term->kanji))
+                throw new \Exception(sprintf(
+                    'Number of hanzi & readings not equal. "%s" & "%s".',
+                    $term->kanji,
+                    implode(' ', $reading),
+                ));
+
+            $term->kanji = implode(' ', array_map(
+                fn(string $hanzi, string $pinyin) => "{$hanzi}[{$pinyin}]",
+                Japanese::getKanjiList($term->kanji),
+                $reading,
+            ));
+        }
+
+        return $note;
+    }
+
+    /** @return mixed[] */
+    public function toAnki(): array
+    {
+        return array_merge(parent::toAnki(), [
+            'fields' => [
+                'VocabKanji' => join('｜', array_map(
+                    fn(Term $x) => $x->getKanji(),
+                    $this->terms,
+                )),
+                'VocabFurigana' => join('｜', array_map(
+                    fn(Term $x) => $x->getReading() ?? '＿',
+                    $this->terms,
+                )),
+                'VocabDef' => join("<br>\n", array_map(
+                    fn(Term $x) => $x->toAnkiVocabDef(),
+                    $this->terms,
+                )),
+            ],
+        ]);
+    }
+
+
+    // ---------------------------------------------------- Derived methods ---
+
+    public function hasTerm(string $kanji): bool
+    {
+        foreach ($this->terms as $term) {
+            if ($term->kanji == $kanji) return true;
+        }
+
+        return false;
+    }
+
+    public function isSentKanjiHighlighted(): bool
+    {
+        return str_contains(
+            $this->fields['SentKanji'],
+            self::HIGHLIGHT_ATTR_KANJI,
+        );
+    }
+
+    /**
+     * Return an array of strings with the highlighted kanji in the SentKanji
+     * @return list<string>
+     */
+    public function getHighlightedKanji(): array
+    {
+        $ret = [];
+        $matches = [];
+
+        // 1. Get all spans in the text
+        preg_match_all(
+            self::HIGHLIGHT_PATTERN,
+            $this->fields['SentKanji'],
+            $matches,
+            PREG_SET_ORDER,
+        );
+
+        // 2. Check the ones that match with the kanji color
+        foreach ($matches as $match) {
+            if ($match[1] === self::HIGHLIGHT_ATTR_KANJI) {
+                $ret[] = mb_trim($match[2]);
+            }
+        }
+
+        return $ret;
+    }
+}

src/Entity/ChineseTone.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace App\Entity;
+
+enum ChineseTone: int
+{
+    case flat    = 1;
+    case rising  = 2;
+    case dip     = 3;
+    case falling = 4;
+    case neutral = 5;
+
+    public static function fromPinyin(string $pinyin): self
+    {
+        return match (1) {
+            preg_match('/[āēīōūǖĀĒĪŌŪǕ]/u', $pinyin) => self::flat,
+            preg_match('/[áéíóúǘÁÉÍÓÚǗ]/u', $pinyin) => self::rising,
+            preg_match('/[ǎěǐǒǔǚǍĚǏǑǓǙ]/u', $pinyin) => self::dip,
+            preg_match('/[àèìòùǜÀÈÌÒÙǛ]/u', $pinyin) => self::falling,
+            default                                  => self::neutral,
+        };
+    }
+
+    public function getColor(): string
+    {
+        return match ($this) {
+            self::flat    => 'red',
+            self::rising  => 'darkorange',
+            self::dip     => 'forestgreen',
+            self::falling => 'darkcyan',
+            self::neutral => 'inherit',
+        };
+    }
+}

src/Entity/Note.php

@@ -98,8 +98,10 @@ class Note
 
     // -------------------------------------------------- Utility functions ---
 
-    protected static function stringHighlight(string $haystack, string $needle): string
-    {
+    protected static function stringHighlight(
+        string $haystack,
+        string $needle,
+    ): string {
         $replace = sprintf(
             '<span %s>%s</span>',
             self::HIGHLIGHT_ATTR_KANJI,

src/Entity/Term.php

@@ -34,11 +34,14 @@ class Term
         $matchedKanji   = array_map(fn($x) => $x[1],          $matches);
         $matchedReading = array_map(fn($x) => $x[3] ?? $x[1], $matches);
 
+        $isJapanese = preg_match('/^[\x{3040}-\x{309F}]/u', implode('', $matchedReading)) === 1;
+        $separator = $isJapanese ? '' : ' ';
+
         return [
             'kanji'   => join('', $matchedKanji),
-            'reading' => $matchedKanji == $matchedReading
+            'reading' => $matchedKanji === $matchedReading
                 ? null
-                : join('', $matchedReading),
+                : join($separator, $matchedReading),
         ];
     }
 
@@ -149,7 +152,9 @@ class Term
      */
     public static function fromNoteFields(array $fields): array
     {
-        $audios = explode('|', $fields['VocabAudio'] ?? '');
+        // Get audio array with sanitization of wide characters
+        $vocabAudio = str_replace('｜', '|', $fields['VocabAudio'] ?? '');
+        $audios = explode('|', $vocabAudio);
 
         // -------------------- Trying to extract it with the modern syntax ---
         // 言葉: word

src/Service/AnkiService.php

@@ -2,6 +2,7 @@
 
 namespace App\Service;
 
+use App\Entity\ChineseSentenceNote;
 use App\Entity\KoreanProductionNote;
 use App\Entity\KoreanSentenceNote;
 use App\Entity\Note;
@@ -99,7 +100,7 @@ class AnkiService
      * @param class-string<T> $class
      * @return list<T>
      */
-    public function getAllFromClass(string $class, ?bool $isSuspended): array
+    public function getAllFromClass(string $class, ?bool $isSuspended = null): array
     {
         $ids = $this->getAllIdsFromClass($class, $isSuspended);
         return $this->getNotes($ids);
@@ -154,6 +155,7 @@ class AnkiService
             SentenceListeningNote::MODEL_NAME => SentenceListeningNote::fromAnki($noteInfo),
             KoreanSentenceNote::MODEL_NAME    => KoreanSentenceNote::fromAnki($noteInfo),
             KoreanProductionNote::MODEL_NAME  => KoreanProductionNote::fromAnki($noteInfo),
+            ChineseSentenceNote::MODEL_NAME   => ChineseSentenceNote::fromAnki($noteInfo),
             default => throw new \Exception(sprintf(
                 'Unrecognized Note "%s" of type "%s"',
                 $noteInfo['noteId'],

templates/kanji/grid.html.twig

@@ -149,6 +149,11 @@
     |
     {% set sn_kanken = char_info.lists.sn|intersect_key(char_info.lists.kanken)|length %}
     {{ sn_kanken }}/{{ total_kanken }} ({{ ((sn_kanken/total_kanken) * 100)|number_format(2) }}%)
+    |
+    {% set sln = char_info.lists.sln|length %}
+    {% set sn = char_info.lists.sn|length %}
+    {{ sln }} / {{ sn }}
+    ({{ ((sln/sn) * 100)|number_format(2) }}%)
   </div>
 </div>
 {% endblock %}

templates/kanji/view.html.twig

@@ -107,6 +107,30 @@
             CHISE
           </a>
         </li>
+        <li>
+          <a
+            class="text-primary"
+            href="{{ 'https://filmot.com/search/' ~ char|url_encode ~ '/1?' ~ {
+              gridView: 1,
+              searchManualSubs: 1,
+              lang: 'zh',
+            }|url_encode }}"
+          >
+            Filmot ZH
+          </a>
+        </li>
+        <li>
+          <a
+            class="text-primary"
+            href="{{ 'https://filmot.com/search/' ~ char|url_encode ~ '/1?' ~ {
+              gridView: 1,
+              searchManualSubs: 1,
+              lang: 'ja',
+            }|url_encode }}"
+          >
+            Filmot JA
+          </a>
+        </li>
       </ul>
     </div>
   </div>