From d0e11c323ee5567b0ce53739d8bb107d37e3de4a Mon Sep 17 00:00:00 2001 From: Joshua Ramon Enslin Date: Fri, 18 Nov 2022 00:26:23 +0100 Subject: [PATCH] Further modularize fetching of translations, add new class NodaBatchInserter for batch inserting translations --- src/NodaBatchInserter.php | 118 ++++++++++++++++++++++++++++++++++++ src/NodaWikidataFetcher.php | 101 ++++++++++++++---------------- 2 files changed, 163 insertions(+), 56 deletions(-) create mode 100644 src/NodaBatchInserter.php diff --git a/src/NodaBatchInserter.php b/src/NodaBatchInserter.php new file mode 100644 index 0000000..ab04189 --- /dev/null +++ b/src/NodaBatchInserter.php @@ -0,0 +1,118 @@ + + */ +declare(strict_types = 1); + +/** + * This file handles the insertion of new entries to a noda DB in batch. + */ +final class NodaBatchInserter { + /** + * Inserts a batch of actor translations. + * + * @param MDMysqli $mysqli_noda DB connection. + * @param array $toInsert List of actor translations to enter. + * + * @return void + */ + public static function insertPersinstTranslations(MDMysqli $mysqli_noda, array $toInsert):void { + + $mysqli_noda->autocommit(false); + $insertStmt = $mysqli_noda->do_prepare("CALL `nodaInsertPersinstTranslation`(?, ?, ?, ?, ?)"); + + foreach ($toInsert as $values) { + + try { + $insertStmt->bind_param("issss", $values['persinst_id'], + $values['lang'], + $values['name'], + $values['description'], + $values['link']); + $insertStmt->execute(); + } + catch (MDMysqliInvalidEncodingError $e) { + } + + } + + $mysqli_noda->commit(); + $insertStmt->close(); + + $mysqli_noda->autocommit(true); + + } + + /** + * Inserts a batch of place translations. + * + * @param MDMysqli $mysqli_noda DB connection. + * @param array $toInsert List of place translations to enter. + * + * @return void + */ + public static function insertPlaceTranslations(MDMysqli $mysqli_noda, array $toInsert):void { + + $mysqli_noda->autocommit(false); + $insertStmt = $mysqli_noda->do_prepare("CALL `nodaInsertOrtTranslation`(?, ?, ?, ?, ?)"); + + foreach ($toInsert as $values) { + + try { + $insertStmt->bind_param("issss", $values['ort_id'], + $values['lang'], + $values['name'], + $values['description'], + $values['link']); + $insertStmt->execute(); + } + catch (MDMysqliInvalidEncodingError $e) { + } + + } + + $mysqli_noda->commit(); + $insertStmt->close(); + + $mysqli_noda->autocommit(true); + + } + + /** + * Inserts a batch of tag translations. + * + * @param MDMysqli $mysqli_noda DB connection. + * @param array $toInsert List of tag translations to enter. + * + * @return void + */ + public static function insertTagTranslations(MDMysqli $mysqli_noda, array $toInsert):void { + + $mysqli_noda->autocommit(false); + $insertStmt = $mysqli_noda->do_prepare("CALL nodaInsertTagTranslation(?, ?, ?, ?, ?)"); + + foreach ($toInsert as $values) { + + try { + $insertStmt->bind_param("issss", $values['tag_id'], + $values['lang'], + $values['name'], + $values['description'], + $values['link']); + $insertStmt->execute(); + } + catch (MDMysqliInvalidEncodingError $e) { + } + + } + + $mysqli_noda->commit(); + $insertStmt->close(); + + $mysqli_noda->autocommit(true); + + } +} diff --git a/src/NodaWikidataFetcher.php b/src/NodaWikidataFetcher.php index f5b50fe..ff5d7e0 100644 --- a/src/NodaWikidataFetcher.php +++ b/src/NodaWikidataFetcher.php @@ -938,37 +938,34 @@ final class NodaWikidataFetcher { /** * Function for fetching translations from Wikipedia, based on Wikidata information. * - * @param array $data Entity fetched from wikidata. - * @param integer $persinst_id Actor ID. + * @param array $data Entity fetched from wikidata. + * @param integer $persinst_id Actor ID. + * @param string[] $checkForLangs Languages to check for. Defaults to all + * languages generally loaded by the wikidata fetcher. * * @return void */ - public function getWikidataTranslationsForPersinst(array $data, int $persinst_id):void { + public function getWikidataTranslationsForPersinst(array $data, int $persinst_id, array $checkForLangs = self::LANGUAGES_TO_CHECK):void { - if (empty($translations = self::listTranslationsFromWikidataWikipedia(self::LANGUAGES_TO_CHECK, $data))) { + if (empty($translations = self::listTranslationsFromWikidataWikipedia($checkForLangs, $data))) { return; } - $insertStmt = $this->_mysqli_noda->do_prepare("CALL nodaInsertPersinstTranslation(?, ?, ?, ?, ?)"); - - $this->_mysqli_noda->autocommit(false); + $toInsert = []; foreach ($translations as $lang => $values) { - try { - $insertStmt->bind_param("issss", $persinst_id, $lang, - $values['label'], $values['description'], $values['link']); - $insertStmt->execute(); - } - catch (MDMysqliInvalidEncodingError $e) { - } + $toInsert[] = [ + 'persinst_id' => $persinst_id, + 'lang' => $lang, + 'name' => $values['label'], + 'description' => $values['description'], + 'link' => $values['link'], + ]; } - $this->_mysqli_noda->commit(); - $this->_mysqli_noda->autocommit(true); - - $insertStmt->close(); + NodaBatchInserter::insertPersinstTranslations($this->_mysqli_noda, $toInsert); } @@ -1265,38 +1262,34 @@ final class NodaWikidataFetcher { /** * Function for fetching translations from wikidata. * - * @param array $data Entity data fetched from wikidata. - * @param integer $ort_id Place ID. + * @param array $data Entity data fetched from wikidata. + * @param integer $ort_id Place ID. + * @param string[] $checkForLangs Languages to check for. Defaults to all + * languages generally loaded by the wikidata fetcher. * * @return void */ - public function getWikidataTranslationsForPlace(array $data, int $ort_id) { + public function getWikidataTranslationsForPlace(array $data, int $ort_id, array $checkForLangs = self::LANGUAGES_TO_CHECK):void { - if (empty($translations = self::listTranslationsFromWikidataWikipedia(self::LANGUAGES_TO_CHECK, $data))) { + if (empty($translations = self::listTranslationsFromWikidataWikipedia($checkForLangs, $data))) { return; } - $insertStmt = $this->_mysqli_noda->do_prepare("CALL `nodaInsertOrtTranslation`(?, ?, ?, ?, ?)"); - - $this->_mysqli_noda->autocommit(false); + $toInsert = []; foreach ($translations as $lang => $values) { - try { - $insertStmt->bind_param("issss", $ort_id, $lang, - $values['label'], $values['description'], $values['link']); - $insertStmt->execute(); - } - catch (MDMysqliInvalidEncodingError $e) { - } + $toInsert[] = [ + 'ort_id' => $ort_id, + 'lang' => $lang, + 'name' => $values['label'], + 'description' => $values['description'], + 'link' => $values['link'], + ]; } - $this->_mysqli_noda->commit(); - $this->_mysqli_noda->autocommit(true); - - $insertStmt->close(); - unset($insertStmt); + NodaBatchInserter::insertPlaceTranslations($this->_mysqli_noda, $toInsert); } @@ -1538,20 +1531,20 @@ final class NodaWikidataFetcher { /** * Function for fetching translations from wikidata. * - * @param array $data Entity data fetched from wikidata. - * @param integer $tag_id Tag ID. + * @param array $data Entity data fetched from wikidata. + * @param integer $tag_id Tag ID. + * @param string[] $checkForLangs Languages to check for. Defaults to all + * languages generally loaded by the wikidata fetcher. * * @return void */ - public function getWikidataTranslationsForTag(array $data, int $tag_id) { + public function getWikidataTranslationsForTag(array $data, int $tag_id, array $checkForLangs = self::LANGUAGES_TO_CHECK):void { - if (empty($translations = self::listTranslationsFromWikidataWikipedia(self::LANGUAGES_TO_CHECK, $data))) { + if (empty($translations = self::listTranslationsFromWikidataWikipedia($checkForLangs, $data))) { return; } - $insertStmt = $this->_mysqli_noda->do_prepare("CALL nodaInsertTagTranslation(?, ?, ?, ?, ?)"); - - $this->_mysqli_noda->autocommit(false); + $toInsert = []; foreach ($translations as $lang => $values) { @@ -1564,21 +1557,17 @@ final class NodaWikidataFetcher { $description = $values['description']; } - try { - $insertStmt->bind_param("issss", $tag_id, $lang, - $label, $description, $values['link']); - $insertStmt->execute(); - } - catch (MDMysqliInvalidEncodingError $e) { - } + $toInsert[] = [ + 'tag_id' => $tag_id, + 'lang' => $lang, + 'name' => $label, + 'description' => $description, + 'link' => $values['link'], + ]; } - $this->_mysqli_noda->commit(); - $this->_mysqli_noda->autocommit(true); - - $insertStmt->close(); - unset($insertStmt); + NodaBatchInserter::insertTagTranslations($this->_mysqli_noda, $toInsert); }