museum-digital/MDNodaHelpers
33
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Further modularize fetching of translations, add new class

Browse Source 
NodaBatchInserter for batch inserting translations
This commit is contained in:
Joshua Ramon Enslin 2022-11-18 00:26:23 +01:00
parent b318b5b471
commit d0e11c323e
Signed by: jrenslin
GPG Key ID: 46016F84501B70AE
2 changed files with 163 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
src/NodaBatchInserter.php Normal file
View File

@ -0,0 +1,118 @@
<?PHP
/**
* This file handles the insertion of new entries to a noda DB in batch.
*
* @file
* @author Joshua Ramon Enslin <joshua@museum-digital.de>
*/
declare(strict_types = 1);
/**
* This file handles the insertion of new entries to a noda DB in batch.
*/
final class NodaBatchInserter {
/**
* Inserts a batch of actor translations.
*
* @param MDMysqli $mysqli_noda DB connection.
* @param array<array{persinst_id: int, lang: string, name: string, description: string, link: string}> $toInsert List of actor translations to enter.
*
* @return void
*/
public static function insertPersinstTranslations(MDMysqli $mysqli_noda, array $toInsert):void {
$mysqli_noda->autocommit(false);
$insertStmt = $mysqli_noda->do_prepare("CALL `nodaInsertPersinstTranslation`(?, ?, ?, ?, ?)");
foreach ($toInsert as $values) {
try {
$insertStmt->bind_param("issss", $values['persinst_id'],
$values['lang'],
$values['name'],
$values['description'],
$values['link']);
$insertStmt->execute();
}
catch (MDMysqliInvalidEncodingError $e) {
}
}
$mysqli_noda->commit();
$insertStmt->close();
$mysqli_noda->autocommit(true);
}
/**
* Inserts a batch of place translations.
*
* @param MDMysqli $mysqli_noda DB connection.
* @param array<array{ort_id: int, lang: string, name: string, description: string, link: string}> $toInsert List of place translations to enter.
*
* @return void
*/
public static function insertPlaceTranslations(MDMysqli $mysqli_noda, array $toInsert):void {
$mysqli_noda->autocommit(false);
$insertStmt = $mysqli_noda->do_prepare("CALL `nodaInsertOrtTranslation`(?, ?, ?, ?, ?)");
foreach ($toInsert as $values) {
try {
$insertStmt->bind_param("issss", $values['ort_id'],
$values['lang'],
$values['name'],
$values['description'],
$values['link']);
$insertStmt->execute();
}
catch (MDMysqliInvalidEncodingError $e) {
}
}
$mysqli_noda->commit();
$insertStmt->close();
$mysqli_noda->autocommit(true);
}
/**
* Inserts a batch of tag translations.
*
* @param MDMysqli $mysqli_noda DB connection.
* @param array<array{tag_id: int, lang: string, name: string, description: string, link: string}> $toInsert List of tag translations to enter.
*
* @return void
*/
public static function insertTagTranslations(MDMysqli $mysqli_noda, array $toInsert):void {
$mysqli_noda->autocommit(false);
$insertStmt = $mysqli_noda->do_prepare("CALL nodaInsertTagTranslation(?, ?, ?, ?, ?)");
foreach ($toInsert as $values) {
try {
$insertStmt->bind_param("issss", $values['tag_id'],
$values['lang'],
$values['name'],
$values['description'],
$values['link']);
$insertStmt->execute();
}
catch (MDMysqliInvalidEncodingError $e) {
}
}
$mysqli_noda->commit();
$insertStmt->close();
$mysqli_noda->autocommit(true);
}
}

101
src/NodaWikidataFetcher.php
View File

@ -938,37 +938,34 @@ final class NodaWikidataFetcher {
/**
* Function for fetching translations from Wikipedia, based on Wikidata information.
*
* @param array<mixed> $data Entity fetched from wikidata.
* @param integer $persinst_id Actor ID.
* @param array<mixed> $data Entity fetched from wikidata.
* @param integer $persinst_id Actor ID.
* @param string[] $checkForLangs Languages to check for. Defaults to all
* languages generally loaded by the wikidata fetcher.
*
* @return void
*/
public function getWikidataTranslationsForPersinst(array $data, int $persinst_id):void {
public function getWikidataTranslationsForPersinst(array $data, int $persinst_id, array $checkForLangs = self::LANGUAGES_TO_CHECK):void {
if (empty($translations = self::listTranslationsFromWikidataWikipedia(self::LANGUAGES_TO_CHECK, $data))) {
if (empty($translations = self::listTranslationsFromWikidataWikipedia($checkForLangs, $data))) {
return;
}
$insertStmt = $this->_mysqli_noda->do_prepare("CALL nodaInsertPersinstTranslation(?, ?, ?, ?, ?)");
$this->_mysqli_noda->autocommit(false);
$toInsert = [];
foreach ($translations as $lang => $values) {
try {
$insertStmt->bind_param("issss", $persinst_id, $lang,
$values['label'], $values['description'], $values['link']);
$insertStmt->execute();
}
catch (MDMysqliInvalidEncodingError $e) {
}
$toInsert[] = [
'persinst_id' => $persinst_id,
'lang' => $lang,
'name' => $values['label'],
'description' => $values['description'],
'link' => $values['link'],
];
}
$this->_mysqli_noda->commit();
$this->_mysqli_noda->autocommit(true);
$insertStmt->close();
NodaBatchInserter::insertPersinstTranslations($this->_mysqli_noda, $toInsert);
}
@ -1265,38 +1262,34 @@ final class NodaWikidataFetcher {
/**
* Function for fetching translations from wikidata.
*
* @param array<mixed> $data Entity data fetched from wikidata.
* @param integer $ort_id Place ID.
* @param array<mixed> $data Entity data fetched from wikidata.
* @param integer $ort_id Place ID.
* @param string[] $checkForLangs Languages to check for. Defaults to all
* languages generally loaded by the wikidata fetcher.
*
* @return void
*/
public function getWikidataTranslationsForPlace(array $data, int $ort_id) {
public function getWikidataTranslationsForPlace(array $data, int $ort_id, array $checkForLangs = self::LANGUAGES_TO_CHECK):void {
if (empty($translations = self::listTranslationsFromWikidataWikipedia(self::LANGUAGES_TO_CHECK, $data))) {
if (empty($translations = self::listTranslationsFromWikidataWikipedia($checkForLangs, $data))) {
return;
}
$insertStmt = $this->_mysqli_noda->do_prepare("CALL `nodaInsertOrtTranslation`(?, ?, ?, ?, ?)");
$this->_mysqli_noda->autocommit(false);
$toInsert = [];
foreach ($translations as $lang => $values) {
try {
$insertStmt->bind_param("issss", $ort_id, $lang,
$values['label'], $values['description'], $values['link']);
$insertStmt->execute();
}
catch (MDMysqliInvalidEncodingError $e) {
}
$toInsert[] = [
'ort_id' => $ort_id,
'lang' => $lang,
'name' => $values['label'],
'description' => $values['description'],
'link' => $values['link'],
];
}
$this->_mysqli_noda->commit();
$this->_mysqli_noda->autocommit(true);
$insertStmt->close();
unset($insertStmt);
NodaBatchInserter::insertPlaceTranslations($this->_mysqli_noda, $toInsert);
}
@ -1538,20 +1531,20 @@ final class NodaWikidataFetcher {
/**
* Function for fetching translations from wikidata.
*
* @param array<mixed> $data Entity data fetched from wikidata.
* @param integer $tag_id Tag ID.
* @param array<mixed> $data Entity data fetched from wikidata.
* @param integer $tag_id Tag ID.
* @param string[] $checkForLangs Languages to check for. Defaults to all
* languages generally loaded by the wikidata fetcher.
*
* @return void
*/
public function getWikidataTranslationsForTag(array $data, int $tag_id) {
public function getWikidataTranslationsForTag(array $data, int $tag_id, array $checkForLangs = self::LANGUAGES_TO_CHECK):void {
if (empty($translations = self::listTranslationsFromWikidataWikipedia(self::LANGUAGES_TO_CHECK, $data))) {
if (empty($translations = self::listTranslationsFromWikidataWikipedia($checkForLangs, $data))) {
return;
}
$insertStmt = $this->_mysqli_noda->do_prepare("CALL nodaInsertTagTranslation(?, ?, ?, ?, ?)");
$this->_mysqli_noda->autocommit(false);
$toInsert = [];
foreach ($translations as $lang => $values) {
@ -1564,21 +1557,17 @@ final class NodaWikidataFetcher {
$description = $values['description'];
}
try {
$insertStmt->bind_param("issss", $tag_id, $lang,
$label, $description, $values['link']);
$insertStmt->execute();
}
catch (MDMysqliInvalidEncodingError $e) {
}
$toInsert[] = [
'tag_id' => $tag_id,
'lang' => $lang,
'name' => $label,
'description' => $description,
'link' => $values['link'],
];
}
$this->_mysqli_noda->commit();
$this->_mysqli_noda->autocommit(true);
$insertStmt->close();
unset($insertStmt);
NodaBatchInserter::insertTagTranslations($this->_mysqli_noda, $toInsert);
}