Compare commits
3 Commits
b7ec7623b2
...
da9d4d344a
Author | SHA1 | Date |
---|---|---|
|
da9d4d344a | |
|
e2c178e998 | |
|
cfaafb58e1 |
|
@ -6,6 +6,8 @@ use App\Entity\SentenceListeningNote;
|
|||
use App\Entity\SentenceNote;
|
||||
use App\Entity\Term;
|
||||
use App\Service\AnkiService;
|
||||
use App\Utils\Japanese;
|
||||
use App\Utils\Progress;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
|
@ -88,55 +90,55 @@ class CreateProductionCommand extends Command
|
|||
}
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
/** @return list<SentenceNote> */
|
||||
private function getAllSentenceNotes(): array
|
||||
{
|
||||
printf('Getting all SentenceCards...');
|
||||
printf('Getting all SentenceNote...');
|
||||
$allIds = $this->ankiService->getAllSentenceNoteIds();
|
||||
$allNotes = $this->ankiService->getNotes($allIds);
|
||||
printf(" OK (%d)\n", count($allNotes));
|
||||
return $allNotes;
|
||||
}
|
||||
|
||||
printf('Getting all SentenceCards...');
|
||||
/** @return list<SentenceListeningNote> */
|
||||
private function getAllSentenceListeningNotes(): array
|
||||
{
|
||||
printf('Getting all SentenceListeningNote...');
|
||||
$allListeningIds = $this->ankiService->getAllSentenceListeningNoteIds();
|
||||
$allListeningNotes = $this->ankiService->getNotes($allListeningIds);
|
||||
printf(" OK (%d)\n", count($allListeningNotes));
|
||||
return $allListeningNotes;
|
||||
}
|
||||
|
||||
printf('Indexing all terms...');
|
||||
$knownTerms = [];
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$allSentenceNotes = $this->getAllSentenceNotes();
|
||||
$allListeningNotes = $this->getAllSentenceListeningNotes();
|
||||
|
||||
// --------- Getting list<SentenceNote> into array<TermKanji, Term> ---
|
||||
|
||||
$allTerms = [];
|
||||
$knownKanji = [];
|
||||
$termCounts = [];
|
||||
foreach ($allNotes as $note) {
|
||||
if (!$note instanceof SentenceNote) throw new \Exception(sprintf(
|
||||
'Expected SentenceNote, got %s',
|
||||
$note::class,
|
||||
));
|
||||
|
||||
printf('Indexing all terms...');
|
||||
foreach ($allSentenceNotes as $note) {
|
||||
foreach ($note->getTerms() as &$term) {
|
||||
assert($term instanceof Term);
|
||||
|
||||
if (key_exists($term->getKanji(), $knownTerms)) continue;
|
||||
if (key_exists($term->getKanji(), $allTerms)) continue;
|
||||
$termCounts[$term->getKanji()] = 0;
|
||||
$knownTerms[$term->getKanji()] = &$term;
|
||||
$allTerms[$term->getKanji()] = &$term;
|
||||
foreach (self::extractKanji($term->getKanji()) as $kanji) {
|
||||
$knownKanji[$kanji] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
printf(" OK (%d)\n", count($knownTerms));
|
||||
printf(" OK (%d)\n", count($allTerms));
|
||||
|
||||
$total = count($knownTerms);
|
||||
$i = 0;
|
||||
foreach ($allNotes as $note) {
|
||||
$i += 1;
|
||||
if ($i % 12 === 0 or $i === $total) {
|
||||
printf(
|
||||
"\33[2K\r% 7d/% 7d | %.2f GiB | Getting frequencies",
|
||||
$i,
|
||||
$total,
|
||||
memory_get_usage() / 1024 / 1024 / 1024
|
||||
);
|
||||
}
|
||||
|
||||
assert($note instanceof SentenceNote);
|
||||
$progress = new Progress('Getting frequenciees', count($allSentenceNotes));
|
||||
foreach ($allSentenceNotes as $note) {
|
||||
$progress->tick();
|
||||
|
||||
$sentKanji = str_replace(
|
||||
"\u{200E}",
|
||||
|
@ -144,83 +146,82 @@ class CreateProductionCommand extends Command
|
|||
strip_tags($note->getFields()['SentKanji'])
|
||||
);
|
||||
|
||||
//foreach ($knownTerms as &$term) {
|
||||
// assert($term instanceof Term);
|
||||
|
||||
// if (str_contains($sentKanji, $term->getKanji())) {
|
||||
// $termCounts[$term->getKanji()] += 1;
|
||||
// }
|
||||
//}
|
||||
|
||||
foreach ($knownKanji as $kanji => &$count) {
|
||||
if (str_contains($sentKanji, $kanji)) {
|
||||
$count++;
|
||||
}
|
||||
if (str_contains($sentKanji, $kanji)) $count++;
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
$seenKanji = [];
|
||||
//uksort($knownTerms, function ($a, $b) {
|
||||
// //return strlen(self::getOnlyKanji($b)) <=> strlen(self::getOnlyKanji($a)); // descending order
|
||||
// return strlen($b) <=> strlen($a); // ascending order
|
||||
//});
|
||||
|
||||
$seenKanji = $this->ankiService->getKnownSlnKanjiCounts();
|
||||
printf('Rating terms...');
|
||||
foreach ($knownTerms as $term) {
|
||||
$termKanji = self::getOnlyKanji($term->getKanji());
|
||||
$weight = 1 / max(mb_strlen($termKanji), 1);
|
||||
foreach ($allTerms as $key => $term) {
|
||||
$count = Japanese::kanjiDiff(
|
||||
$term->getKanji(),
|
||||
array_keys($seenKanji),
|
||||
);
|
||||
|
||||
// First pass: Calculate the weight
|
||||
foreach ($knownKanji as $kanji => $count) {
|
||||
if (str_contains($termKanji, $kanji)) {
|
||||
$termCounts[$term->getKanji()] += ceil($count * $weight);
|
||||
}
|
||||
}
|
||||
if ($count <= 0) {
|
||||
unset($allTerms[$key]);
|
||||
unset($termCounts[$key]);
|
||||
continue;
|
||||
}
|
||||
|
||||
arsort($termCounts);
|
||||
$termCounts[$term->getKanji()] = $count;
|
||||
}
|
||||
uksort($termCounts, function ($a, $b) {
|
||||
$aLen = mb_strlen(Japanese::getOnlyKanji($a));
|
||||
$bLen = mb_strlen(Japanese::getOnlyKanji($b));
|
||||
return $bLen <=> $aLen;
|
||||
});
|
||||
asort($termCounts);
|
||||
printf(" OK\n");
|
||||
|
||||
// Have into account the ones that have already been created.
|
||||
// This will not only skip them but take into account the kanjis they
|
||||
// have.
|
||||
// This will not only skip them but also update the general array for
|
||||
// already seen kanji.
|
||||
$seenKanji = [];
|
||||
printf('Filtering out terms with no new kanji...');
|
||||
// First pass: Get the list of the kanji we've seen
|
||||
foreach ($allListeningNotes as $listeningNote) {
|
||||
assert($listeningNote instanceof SentenceListeningNote);
|
||||
|
||||
$termKanji = self::getOnlyKanji($listeningNote->getTerm()->getKanji());
|
||||
self::kanjiDiff($seenKanji, $termKanji);
|
||||
}
|
||||
|
||||
// Second pass: Remove terms with no new kanji at all
|
||||
foreach ($termCounts as $term => $count) {
|
||||
$termKanji = self::getOnlyKanji($term);
|
||||
|
||||
// Second pass: Penalize terms with no new kanji at all
|
||||
if (!self::kanjiDiff($seenKanji, $termKanji)) {
|
||||
unset($termCounts[$term]);
|
||||
//unset($knownTerms[$term->getKanji()]);
|
||||
//$termCounts[$term->getKanji()] = 0;
|
||||
unset($allTerms[$term]);
|
||||
}
|
||||
}
|
||||
printf(" OK\n");
|
||||
|
||||
arsort($termCounts);
|
||||
asort($termCounts);
|
||||
printf("\n");
|
||||
|
||||
|
||||
$newNotesCount = intval($input->getArgument('count'));
|
||||
|
||||
foreach ($termCounts as $term => $count) {
|
||||
if ($newNotesCount <= 0) break;
|
||||
|
||||
// FIXME: This shouldn't happen at all
|
||||
if (!$allTerms[$term] instanceof Term) continue;
|
||||
|
||||
$termKanji = self::getOnlyKanji($term);
|
||||
printf("%s: %d\n", $term, $count);
|
||||
|
||||
$len = mb_strlen($termKanji);
|
||||
for ($i = 0; $i < $len; $i++) {
|
||||
$iKanji = mb_substr($termKanji, $i, 1);
|
||||
printf(" - %s: %0.2f\n", $iKanji, $knownKanji[$iKanji] / $len);
|
||||
}
|
||||
//$len = mb_strlen($termKanji);
|
||||
//for ($i = 0; $i < $len; $i++) {
|
||||
// $iKanji = mb_substr($termKanji, $i, 1);
|
||||
// printf(" - %s: %0.2f\n", $iKanji, $knownKanji[$iKanji] / $len);
|
||||
//}
|
||||
|
||||
$this->createProductionNoteFromTerm($knownTerms[$term]);
|
||||
$this->createProductionNoteFromTerm($allTerms[$term]);
|
||||
$newNotesCount -= 1;
|
||||
};
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ namespace App\Service;
|
|||
use App\Entity\Note;
|
||||
use App\Entity\SentenceListeningNote;
|
||||
use App\Entity\SentenceNote;
|
||||
use App\Utils\Japanese;
|
||||
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||
|
||||
class AnkiService
|
||||
|
@ -131,4 +132,26 @@ class AnkiService
|
|||
$this->request('updateNoteFields', ['note' => $note->toAnki()]);
|
||||
$this->request('guiBrowse', ['query' => 'nid:' . $note->getId()]);
|
||||
}
|
||||
|
||||
/** @return array<string, int> */
|
||||
public function getKnownSlnKanjiCounts(): array
|
||||
{
|
||||
$allListeningIds = $this->getAllSentenceListeningNoteIds();
|
||||
$ret = [];
|
||||
|
||||
foreach ($this->getNotes($allListeningIds) as $slNote) {
|
||||
assert($slNote instanceof SentenceListeningNote);
|
||||
|
||||
$termKanji = Japanese::getOnlyKanji($slNote->getTerm()->getKanji());
|
||||
$len = mb_strlen($termKanji);
|
||||
for ($i = 0; $i < $len; $i++) {
|
||||
$kanji = mb_substr($termKanji, $i, 1);
|
||||
|
||||
$ret[$kanji] ??= 0;
|
||||
$ret[$kanji]++;
|
||||
}
|
||||
}
|
||||
|
||||
return $ret;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,38 @@
|
|||
<?php
|
||||
|
||||
namespace App\Utils;
|
||||
|
||||
class Japanese
|
||||
{
|
||||
public static function getOnlyKanji(string $str): string
|
||||
{
|
||||
return preg_replace('/[^\p{Script=Han}]/u', '', $str);
|
||||
}
|
||||
|
||||
/** Get the number of kanji of a string that are not in the given set
|
||||
* of kanji
|
||||
*
|
||||
* Only kanji are considere, not katakana, hiragana or any other symbols.
|
||||
*
|
||||
* @param list<string> $kanjiSet
|
||||
*/
|
||||
public static function kanjiDiff(string $str, array $kanjiSet): int
|
||||
{
|
||||
$ret = 0;
|
||||
|
||||
$strKanji = self::getOnlyKanji($str);
|
||||
$len = mb_strlen($strKanji);
|
||||
for ($i = 0; $i < $len; $i++) {
|
||||
$kanji = mb_substr($strKanji, $i, 1);
|
||||
|
||||
if (!array_search($kanji, $kanjiSet)) {
|
||||
$ret++;
|
||||
}
|
||||
}
|
||||
|
||||
//dump($str, $strKanji, $ret);
|
||||
//echo "\n";
|
||||
|
||||
return $ret;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,26 @@
|
|||
<?php
|
||||
|
||||
namespace App\Utils;
|
||||
|
||||
class Progress
|
||||
{
|
||||
function __construct(
|
||||
private string $message,
|
||||
private int $total,
|
||||
private int $speed = 12,
|
||||
private int $i = 0,
|
||||
) {}
|
||||
|
||||
public function tick()
|
||||
{
|
||||
$this->i += 1;
|
||||
if ($this->i % $this->speed === 0 or $this->i === $this->total) {
|
||||
printf(
|
||||
"\33[2K\r% 7d/% 7d | %.2f GiB | {$this->message}",
|
||||
$this->i,
|
||||
$this->total,
|
||||
memory_get_usage() / 1024 / 1024 / 1024
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue