museum-digital/csvxml
35
0 Fork 0
Code Issues 3 Pull Requests Releases Activity

Use papaparse for parsing CSV

Browse Source
This commit is contained in:
jrenslin committed 2022-11-13 02:20:02 +01:00
1 parent 8de7ec1809
commit 2a5c158ac7
6 files changed
+53 -76
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

public/assets/js/csvxmlV2.js
+34 -60
View File
@@ -24,59 +24,13 @@ class CsvxmlValidator {
this.fieldList = Object.freeze(fieldList);
const lines = csvRaw.trim().replace("\r\n", "\n").split("\n");
const data = Papa.parse(csvRaw.trim(), {header: true});
let separator;
let delimiter;
if (csvRaw.substr(0, 1) === '"') {
separator = '";"';
delimiter = '"'
}
else {
separator = ';';
delimiter = '';
if (data.errors.length !== 0) {
window.alert(data.errors);
}
// Gets first line
let headersFields = lines.shift();
let headers;
if (delimiter === "") headers = headersFields.split(separator);
else headers = headersFields.substr(1, headersFields.length - 2).split(separator);
let expectedFieldCount = headers.length;
let toValidate = [];
let lineCounter = 1;
for (let line of lines) {
if (delimiter !== '') {
line = line.substr(1, line.length - 2);
}
// Remove fully empty lines (both without content and those with fields set up,
// but without content there.
if (line.length <= headers.length) continue;
let lineContents = {};
let fields = line.split(separator);
if (fields.length !== headers.length) {
this.errors.parsing.push("Number of columns in line " + lineCounter + " does not match number of headers");
}
for (let i = 0, max = fields.length; i < max; i++) {
if (headers[i] === undefined || headers[i] === null) {
this.errors.parsing.push("ERROR parsing line " + lineCounter + "; column " + i);
continue;
}
lineContents[headers[i]] = fields[i];
}
// Skip totally empty lines
if (Object.values(lineContents).join("").length === 0) continue;
toValidate.push(lineContents);
lineCounter++;
}
let toValidate = data.data;
this.toValidate = toValidate;
@@ -577,18 +531,38 @@ class CsvxmlPage {
reader.onload = function() {
// On loading success, check if the upload is valid JSON
console.log("Read file");
// Validate the file
let validator = new CsvxmlValidator(app.fieldListFlat, reader.result);
document.body.classList.remove("loading");
if (validator.isValid() === true) {
alert("Document is valid. Press ok to download.");
app.zipUploadToXml(validator);
function handleValidation() {
// On loading success, check if the upload is valid JSON
console.log("Read file");
// Validate the file
let validator = new CsvxmlValidator(app.fieldListFlat, reader.result);
document.body.classList.remove("loading");
if (validator.isValid() === true) {
alert("Document is valid. Press ok to download.");
app.zipUploadToXml(validator);
}
else {
console.log("Identified invalid upload document");
app.listValidationErrors(validator);
}
}
console.log("Postload papaparse");
if (typeof Papa === "undefined") {
const loadScript = document.createElement("script");
loadScript.setAttribute("src", "assets/js/papaparse/papaparse.min.js");
loadScript.addEventListener('load', function() {
// console.log("Post-loaded OpenLayers");
handleValidation();
}, {passive: true, once: true});
document.body.appendChild(loadScript);
}
else {
console.log("Identified invalid upload document");
app.listValidationErrors(validator);
handleValidation();
}
};
public/assets/js/csvxmlV2.min.js
Vendored
+6 -10
View File
@@ -1,12 +1,6 @@
"use strict";if('serviceWorker'in navigator){console.log("Registering service worker");navigator.serviceWorker.register('/sw.js');}
class CsvxmlValidator{fieldList;toValidate;errors;constructor(fieldList,csvRaw){this.errors={parsing:[],mandatoryTags:[],duplicateInvNos:[],dependentColumns:[],controlledLists:[],mainImageResource:[],};this.fieldList=Object.freeze(fieldList);const lines=csvRaw.trim().replace("\r\n","\n").split("\n");let separator;let delimiter;if(csvRaw.substr(0,1)==='"'){separator='";"';delimiter='"'}
else{separator=';';delimiter='';}
let headersFields=lines.shift();let headers;if(delimiter==="")headers=headersFields.split(separator);else headers=headersFields.substr(1,headersFields.length-2).split(separator);let expectedFieldCount=headers.length;let toValidate=[];let lineCounter=1;for(let line of lines){if(delimiter!==''){line=line.substr(1,line.length-2);}
if(line.length<=headers.length)continue;let lineContents={};let fields=line.split(separator);if(fields.length!==headers.length){this.errors.parsing.push("Number of columns in line "+lineCounter+" does not match number of headers");}
for(let i=0,max=fields.length;i<max;i++){if(headers[i]===undefined||headers[i]===null){this.errors.parsing.push("ERROR parsing line "+lineCounter+"; column "+i);continue;}
lineContents[headers[i]]=fields[i];}
if(Object.values(lineContents).join("").length===0)continue;toValidate.push(lineContents);lineCounter++;}
this.toValidate=toValidate;if(toValidate.length===0){alert("Error: No lines of content identified");}
class CsvxmlValidator{fieldList;toValidate;errors;constructor(fieldList,csvRaw){this.errors={parsing:[],mandatoryTags:[],duplicateInvNos:[],dependentColumns:[],controlledLists:[],mainImageResource:[],};this.fieldList=Object.freeze(fieldList);const data=Papa.parse(csvRaw.trim(),{header:true});if(data.errors.length!==0){window.alert(data.errors);}
let toValidate=data.data;this.toValidate=toValidate;if(toValidate.length===0){alert("Error: No lines of content identified");}
this.validate();}
validate(){this.validateMandatoryTagsPresent();this.checkDuplicateInvNos();this.checkDependentColumns();this.checkControlledLists();}
validateMandatoryTagsPresent(){let mandatoryFields=[];for(let fieldName in this.fieldList){if(this.fieldList[fieldName].required===true){mandatoryFields.push(fieldName);}}
@@ -50,8 +44,10 @@ else{runZipping();}}
listValidationErrors(validator){console.log("Listing validation errors");const dialogueContent=document.createElement("div");const headline=document.createElement("h3");headline.textContent=this.tls.validation_errors;const cancelB=document.createElement("a");cancelB.classList.add("icons");cancelB.classList.add("iconsClose");cancelB.classList.add("dialogueCloseX");cancelB.id="dialogueClose";cancelB.textContent="X";cancelB.title="Close";cancelB.href="#"+location.href;cancelB.addEventListener('click',CsvxmlDialogue.closeDialogue);headline.appendChild(cancelB);dialogueContent.appendChild(headline);const domErrorsSection=document.createElement("div");for(let errorType in validator.errors){if(validator.errors[errorType].length===0)continue;const ulHl=document.createElement("h4");ulHl.textContent=this.tls['errors_'+errorType]+" ("+validator.errors[errorType].length+")";ulHl.style.cursor="pointer";domErrorsSection.appendChild(ulHl);const ul=document.createElement("ul");for(let error of validator.errors[errorType]){const li=document.createElement("li");li.textContent=error;ul.appendChild(li);}
ulHl.addEventListener('click',function(){ul.classList.toggle("minimized");});domErrorsSection.appendChild(ul);}
dialogueContent.appendChild(domErrorsSection);const domDlSection=document.createElement("div");const domDlA=document.createElement("span");domDlA.textContent=this.tls.download;domDlA.classList.add("buttonLike");let app=this;domDlA.addEventListener('click',function(){app.zipUploadToXml(validator);});domDlSection.appendChild(domDlA);dialogueContent.appendChild(domDlSection);dialogue=CsvxmlDialogue.drawDialogue(dialogueContent);}
uploadFileForValidation(file){const reader=new FileReader();reader.readAsText(file);let app=this;document.body.classList.add("loading");reader.onload=function(){console.log("Read file");let validator=new CsvxmlValidator(app.fieldListFlat,reader.result);document.body.classList.remove("loading");if(validator.isValid()===true){alert("Document is valid. Press ok to download.");app.zipUploadToXml(validator);}
else{console.log("Identified invalid upload document");app.listValidationErrors(validator);}};reader.onerror=function(){alert(reader.error);};}
uploadFileForValidation(file){const reader=new FileReader();reader.readAsText(file);let app=this;document.body.classList.add("loading");reader.onload=function(){function handleValidation(){console.log("Read file");let validator=new CsvxmlValidator(app.fieldListFlat,reader.result);document.body.classList.remove("loading");if(validator.isValid()===true){alert("Document is valid. Press ok to download.");app.zipUploadToXml(validator);}
else{console.log("Identified invalid upload document");app.listValidationErrors(validator);}}
console.log("Postload papaparse");if(typeof Papa==="undefined"){const loadScript=document.createElement("script");loadScript.setAttribute("src","assets/js/papaparse/papaparse.min.js");loadScript.addEventListener('load',function(){handleValidation();},{passive:true,once:true});document.body.appendChild(loadScript);}
else{handleValidation();}};reader.onerror=function(){alert(reader.error);};}
renderGenHeader(){const header=document.createElement("header");header.id="mainHeader";const logoArea=document.createElement("a");logoArea.id="logoArea";logoArea.href="https://www.museum-digital.org/";const logoImg=document.createElement("img");logoImg.src="assets/img/mdlogo-code-128px.png";logoImg.alt="Logo of museum-digital";logoArea.appendChild(logoImg);const h2=document.createElement("h2");h2.textContent="museum-digital";logoArea.appendChild(h2);header.appendChild(logoArea);const nav=document.createElement("nav");const lAbout=document.createElement("a");lAbout.href="https://en.about.museum-digital.org/about";lAbout.textContent=this.tls.about;nav.appendChild(lAbout);const lContactList=document.createElement("div");const lContact=document.createElement("a");lContact.textContent=this.tls.contact;lContact.href="https://en.about.museum-digital.org/contact/";lContactList.appendChild(lContact);const lContactDiv=document.createElement("div");const lImprint=document.createElement("a");lImprint.textContent=this.tls.imprint;lImprint.href="https://en.about.museum-digital.org/impressum";lContactDiv.appendChild(lImprint);const lPrivacy=document.createElement("a");lPrivacy.textContent=this.tls.privacy_policy;lPrivacy.href="https://en.about.museum-digital.org/privacy/";lContactDiv.appendChild(lPrivacy);lContactList.appendChild(lContactDiv);nav.appendChild(lContactList);const lNews=document.createElement("a")
lNews.textContent=this.tls.news;lNews.href="https://blog.museum-digital.org/";nav.appendChild(lNews);header.appendChild(nav);document.body.appendChild(header);}
renderHeader(){const appHeader=document.createElement("header");const h1=document.createElement("h1");const img=document.createElement("img");img.width="70";img.height="70";img.src="assets/img/mdlogo-csvxml.svg";img.alt="";h1.appendChild(img);const h1Span=document.createElement("span");h1Span.textContent="museum-digital:csvxml";h1.appendChild(h1Span);appHeader.appendChild(h1);document.body.appendChild(appHeader);}
public/assets/js/papaparse/papaparse.min.js
Vendored
+7
View File
@@ -0,0 +1,7 @@
/* @license
Papa Parse
v5.3.2
https://github.com/mholt/PapaParse
License: MIT
*/
!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var r=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),b.BLOB_URL||(b.BLOB_URL=i.createObjectURL(new Blob(["(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=_,t.id=h++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);("boolean"==typeof t.escapeFormulae||t.escapeFormulae instanceof RegExp)&&(o=t.escapeFormulae instanceof RegExp?t.escapeFormulae:/^[=+\-@\t\r].*$/)}();var h=new RegExp(j(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0<e.length,s=!Array.isArray(t[0]);if(n&&_){for(var a=0;a<e.length;a++)0<a&&(r+=m),r+=v(e[a],a);0<t.length&&(r+=y)}for(var o=0;o<t.length;o++){var h=n?e.length:t[o].length,u=!1,f=n?0===Object.keys(t[o]).length:0===t[o].length;if(i&&!n&&(u="greedy"===i?""===t[o].join("").trim():1===t[o].length&&0===t[o][0].length),"greedy"===i&&n){for(var d=[],l=0;l<h;l++){var c=s?e[l]:l;d.push(t[o][c])}u=""===d.join("").trim()}if(!u){for(var p=0;p<h;p++){0<p&&!f&&(r+=m);var g=n&&s?e[p]:p;r+=v(t[o][g],p)}o<t.length-1&&(!i||0<h&&!f)&&(r+=y)}}return r}function v(e,t){if(null==e)return"";if(e.constructor===Date)return JSON.stringify(e).slice(1,25);var i=!1;o&&"string"==typeof e&&o.test(e)&&(e="'"+e,i=!0);var r=e.toString().replace(h,a);return(i=i||!0===n||"function"==typeof n&&n(e,t)||Array.isArray(n)&&n[t]||function(e,t){for(var i=0;i<t.length;i++)if(-1<e.indexOf(t[i]))return!0;return!1}(r,b.BAD_DELIMITERS)||-1<r.indexOf(m)||" "===r.charAt(0)||" "===r.charAt(r.length-1))?s+r+s:r}}};if(b.RECORD_SEP=String.fromCharCode(30),b.UNIT_SEP=String.fromCharCode(31),b.BYTE_ORDER_MARK="\ufeff",b.BAD_DELIMITERS=["\r","\n",'"',b.BYTE_ORDER_MARK],b.WORKERS_SUPPORTED=!n&&!!f.Worker,b.NODE_STREAM_INPUT=1,b.LocalChunkSize=10485760,b.RemoteChunkSize=5242880,b.DefaultDelimiter=",",b.Parser=E,b.ParserHandle=i,b.NetworkStreamer=l,b.FileStreamer=c,b.StringStreamer=p,b.ReadableStreamStreamer=g,f.jQuery){var d=f.jQuery;d.fn.parse=function(o){var i=o.config||{},h=[];return this.each(function(e){if(!("INPUT"===d(this).prop("tagName").toUpperCase()&&"file"===d(this).attr("type").toLowerCase()&&f.FileReader)||!this.files||0===this.files.length)return!0;for(var t=0;t<this.files.length;t++)h.push({file:this.files[t],inputElem:this,instanceConfig:d.extend({},i)})}),e(),this;function e(){if(0!==h.length){var e,t,i,r,n=h[0];if(M(o.before)){var s=o.before(n.file,n.inputElem);if("object"==typeof s){if("abort"===s.action)return e="AbortError",t=n.file,i=n.inputElem,r=s.reason,void(M(o.error)&&o.error({name:e},t,i,r));if("skip"===s.action)return void u();"object"==typeof s.config&&(n.instanceConfig=d.extend(n.instanceConfig,s.config))}else if(Line truncated