museum-digital/csvxml
museum-digital/csvxml
32
0
Fork 0
Code Issues 3 Pull Requests Releases Activity

Use papaparse for parsing CSV

Browse Source
This commit is contained in:
Joshua Ramon Enslin 2022-11-13 02:20:02 +01:00
parent 8de7ec1809
commit 2a5c158ac7
Signed by: jrenslin
GPG Key ID: 46016F84501B70AE
6 changed files with 53 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
public/assets/js/csvxmlV2.js
View File

@ -24,59 +24,13 @@ class CsvxmlValidator {
this.fieldList = Object.freeze(fieldList);
const lines = csvRaw.trim().replace("\r\n", "\n").split("\n");
const data = Papa.parse(csvRaw.trim(), {header: true});
let separator;
let delimiter;
if (csvRaw.substr(0, 1) === '"') {
separator = '";"';
delimiter = '"'
}
else {
separator = ';';
delimiter = '';
if (data.errors.length !== 0) {
window.alert(data.errors);
}
// Gets first line
let headersFields = lines.shift();
let headers;
if (delimiter === "") headers = headersFields.split(separator);
else headers = headersFields.substr(1, headersFields.length - 2).split(separator);
let expectedFieldCount = headers.length;
let toValidate = [];
let lineCounter = 1;
for (let line of lines) {
if (delimiter !== '') {
line = line.substr(1, line.length - 2);
}
// Remove fully empty lines (both without content and those with fields set up,
// but without content there.
if (line.length <= headers.length) continue;
let lineContents = {};
let fields = line.split(separator);
if (fields.length !== headers.length) {
this.errors.parsing.push("Number of columns in line " + lineCounter + " does not match number of headers");
}
for (let i = 0, max = fields.length; i < max; i++) {
if (headers[i] === undefined || headers[i] === null) {
this.errors.parsing.push("ERROR parsing line " + lineCounter + "; column " + i);
continue;
}
lineContents[headers[i]] = fields[i];
}
// Skip totally empty lines
if (Object.values(lineContents).join("").length === 0) continue;
toValidate.push(lineContents);
lineCounter++;
}
let toValidate = data.data;
this.toValidate = toValidate;
@ -577,18 +531,38 @@ class CsvxmlPage {
reader.onload = function() {
// On loading success, check if the upload is valid JSON
console.log("Read file");
// Validate the file
let validator = new CsvxmlValidator(app.fieldListFlat, reader.result);
document.body.classList.remove("loading");
if (validator.isValid() === true) {
alert("Document is valid. Press ok to download.");
app.zipUploadToXml(validator);
function handleValidation() {
// On loading success, check if the upload is valid JSON
console.log("Read file");
// Validate the file
let validator = new CsvxmlValidator(app.fieldListFlat, reader.result);
document.body.classList.remove("loading");
if (validator.isValid() === true) {
alert("Document is valid. Press ok to download.");
app.zipUploadToXml(validator);
}
else {
console.log("Identified invalid upload document");
app.listValidationErrors(validator);
}
}
console.log("Postload papaparse");
if (typeof Papa === "undefined") {
const loadScript = document.createElement("script");
loadScript.setAttribute("src", "assets/js/papaparse/papaparse.min.js");
loadScript.addEventListener('load', function() {
// console.log("Post-loaded OpenLayers");
handleValidation();
}, {passive: true, once: true});
document.body.appendChild(loadScript);
}
else {
console.log("Identified invalid upload document");
app.listValidationErrors(validator);
handleValidation();
}
};

16
public/assets/js/csvxmlV2.min.js vendored
View File

@ -1,12 +1,6 @@
"use strict";if('serviceWorker'in navigator){console.log("Registering service worker");navigator.serviceWorker.register('/sw.js');}
class CsvxmlValidator{fieldList;toValidate;errors;constructor(fieldList,csvRaw){this.errors={parsing:[],mandatoryTags:[],duplicateInvNos:[],dependentColumns:[],controlledLists:[],mainImageResource:[],};this.fieldList=Object.freeze(fieldList);const lines=csvRaw.trim().replace("\r\n","\n").split("\n");let separator;let delimiter;if(csvRaw.substr(0,1)==='"'){separator='";"';delimiter='"'}
else{separator=';';delimiter='';}
let headersFields=lines.shift();let headers;if(delimiter==="")headers=headersFields.split(separator);else headers=headersFields.substr(1,headersFields.length-2).split(separator);let expectedFieldCount=headers.length;let toValidate=[];let lineCounter=1;for(let line of lines){if(delimiter!==''){line=line.substr(1,line.length-2);}
if(line.length<=headers.length)continue;let lineContents={};let fields=line.split(separator);if(fields.length!==headers.length){this.errors.parsing.push("Number of columns in line "+lineCounter+" does not match number of headers");}
for(let i=0,max=fields.length;i<max;i++){if(headers[i]===undefined||headers[i]===null){this.errors.parsing.push("ERROR parsing line "+lineCounter+"; column "+i);continue;}
lineContents[headers[i]]=fields[i];}
if(Object.values(lineContents).join("").length===0)continue;toValidate.push(lineContents);lineCounter++;}
this.toValidate=toValidate;if(toValidate.length===0){alert("Error: No lines of content identified");}
class CsvxmlValidator{fieldList;toValidate;errors;constructor(fieldList,csvRaw){this.errors={parsing:[],mandatoryTags:[],duplicateInvNos:[],dependentColumns:[],controlledLists:[],mainImageResource:[],};this.fieldList=Object.freeze(fieldList);const data=Papa.parse(csvRaw.trim(),{header:true});if(data.errors.length!==0){window.alert(data.errors);}
let toValidate=data.data;this.toValidate=toValidate;if(toValidate.length===0){alert("Error: No lines of content identified");}
this.validate();}
validate(){this.validateMandatoryTagsPresent();this.checkDuplicateInvNos();this.checkDependentColumns();this.checkControlledLists();}
validateMandatoryTagsPresent(){let mandatoryFields=[];for(let fieldName in this.fieldList){if(this.fieldList[fieldName].required===true){mandatoryFields.push(fieldName);}}
@ -50,8 +44,10 @@ else{runZipping();}}
listValidationErrors(validator){console.log("Listing validation errors");const dialogueContent=document.createElement("div");const headline=document.createElement("h3");headline.textContent=this.tls.validation_errors;const cancelB=document.createElement("a");cancelB.classList.add("icons");cancelB.classList.add("iconsClose");cancelB.classList.add("dialogueCloseX");cancelB.id="dialogueClose";cancelB.textContent="X";cancelB.title="Close";cancelB.href="#"+location.href;cancelB.addEventListener('click',CsvxmlDialogue.closeDialogue);headline.appendChild(cancelB);dialogueContent.appendChild(headline);const domErrorsSection=document.createElement("div");for(let errorType in validator.errors){if(validator.errors[errorType].length===0)continue;const ulHl=document.createElement("h4");ulHl.textContent=this.tls['errors_'+errorType]+" ("+validator.errors[errorType].length+")";ulHl.style.cursor="pointer";domErrorsSection.appendChild(ulHl);const ul=document.createElement("ul");for(let error of validator.errors[errorType]){const li=document.createElement("li");li.textContent=error;ul.appendChild(li);}
ulHl.addEventListener('click',function(){ul.classList.toggle("minimized");});domErrorsSection.appendChild(ul);}
dialogueContent.appendChild(domErrorsSection);const domDlSection=document.createElement("div");const domDlA=document.createElement("span");domDlA.textContent=this.tls.download;domDlA.classList.add("buttonLike");let app=this;domDlA.addEventListener('click',function(){app.zipUploadToXml(validator);});domDlSection.appendChild(domDlA);dialogueContent.appendChild(domDlSection);dialogue=CsvxmlDialogue.drawDialogue(dialogueContent);}
uploadFileForValidation(file){const reader=new FileReader();reader.readAsText(file);let app=this;document.body.classList.add("loading");reader.onload=function(){console.log("Read file");let validator=new CsvxmlValidator(app.fieldListFlat,reader.result);document.body.classList.remove("loading");if(validator.isValid()===true){alert("Document is valid. Press ok to download.");app.zipUploadToXml(validator);}
else{console.log("Identified invalid upload document");app.listValidationErrors(validator);}};reader.onerror=function(){alert(reader.error);};}
uploadFileForValidation(file){const reader=new FileReader();reader.readAsText(file);let app=this;document.body.classList.add("loading");reader.onload=function(){function handleValidation(){console.log("Read file");let validator=new CsvxmlValidator(app.fieldListFlat,reader.result);document.body.classList.remove("loading");if(validator.isValid()===true){alert("Document is valid. Press ok to download.");app.zipUploadToXml(validator);}
else{console.log("Identified invalid upload document");app.listValidationErrors(validator);}}
console.log("Postload papaparse");if(typeof Papa==="undefined"){const loadScript=document.createElement("script");loadScript.setAttribute("src","assets/js/papaparse/papaparse.min.js");loadScript.addEventListener('load',function(){handleValidation();},{passive:true,once:true});document.body.appendChild(loadScript);}
else{handleValidation();}};reader.onerror=function(){alert(reader.error);};}
renderGenHeader(){const header=document.createElement("header");header.id="mainHeader";const logoArea=document.createElement("a");logoArea.id="logoArea";logoArea.href="https://www.museum-digital.org/";const logoImg=document.createElement("img");logoImg.src="assets/img/mdlogo-code-128px.png";logoImg.alt="Logo of museum-digital";logoArea.appendChild(logoImg);const h2=document.createElement("h2");h2.textContent="museum-digital";logoArea.appendChild(h2);header.appendChild(logoArea);const nav=document.createElement("nav");const lAbout=document.createElement("a");lAbout.href="https://en.about.museum-digital.org/about";lAbout.textContent=this.tls.about;nav.appendChild(lAbout);const lContactList=document.createElement("div");const lContact=document.createElement("a");lContact.textContent=this.tls.contact;lContact.href="https://en.about.museum-digital.org/contact/";lContactList.appendChild(lContact);const lContactDiv=document.createElement("div");const lImprint=document.createElement("a");lImprint.textContent=this.tls.imprint;lImprint.href="https://en.about.museum-digital.org/impressum";lContactDiv.appendChild(lImprint);const lPrivacy=document.createElement("a");lPrivacy.textContent=this.tls.privacy_policy;lPrivacy.href="https://en.about.museum-digital.org/privacy/";lContactDiv.appendChild(lPrivacy);lContactList.appendChild(lContactDiv);nav.appendChild(lContactList);const lNews=document.createElement("a")
lNews.textContent=this.tls.news;lNews.href="https://blog.museum-digital.org/";nav.appendChild(lNews);header.appendChild(nav);document.body.appendChild(header);}
renderHeader(){const appHeader=document.createElement("header");const h1=document.createElement("h1");const img=document.createElement("img");img.width="70";img.height="70";img.src="assets/img/mdlogo-csvxml.svg";img.alt="";h1.appendChild(img);const h1Span=document.createElement("span");h1Span.textContent="museum-digital:csvxml";h1.appendChild(h1Span);appHeader.appendChild(h1);document.body.appendChild(appHeader);}

7
public/assets/js/papaparse/papaparse.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

4
public/index.htm
View File

@ -4,7 +4,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
<link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000002" />
<link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000003" />
<link rel="manifest" href="/manifest.json" />
<meta name="theme-color" content="#aa4400" />
@ -27,7 +27,7 @@
</head>
<body class="loading">
<script src="assets/js/csvxmlV2.min.js?v000002" type="text/javascript" async></script>
<script src="assets/js/csvxmlV2.min.js?v000003" type="text/javascript" async></script>
</body>
</html>

4
public/sw.js
View File

@ -4,8 +4,8 @@ const urlsToCache = [
'/index.htm',
'/assets/js/jszip/dist/jszip.min.js',
'/assets/js/csvxmlV2.min.js',
'/assets/css/csvxml.min.css',
'/assets/js/csvxmlV2.min.js?v000003',
'/assets/css/csvxml.min.css?v000003',
'/assets/fonts/SourceSansPro-Regular.woff2',

4
scripts/compile.php
View File

@ -76,7 +76,7 @@ function generateAppShell():string {
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
<link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000002" />
<link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000003" />
<link rel="manifest" href="/manifest.json" />
<meta name="theme-color" content="#aa4400" />
@ -99,7 +99,7 @@ function generateAppShell():string {
</head>
<body class="loading">
<script src="assets/js/csvxmlV2.min.js?v000002" type="text/javascript" async></script>
<script src="assets/js/csvxmlV2.min.js?v000003" type="text/javascript" async></script>
</body>
</html>';