Use papaparse for parsing CSV

public/assets/js/csvxmlV2.js

@@ -24,59 +24,13 @@ class CsvxmlValidator {
 
         this.fieldList = Object.freeze(fieldList);
 
-        const lines = csvRaw.trim().replace("\r\n", "\n").split("\n");
+        const data = Papa.parse(csvRaw.trim(), {header: true});
 
-        let separator;
+        if (data.errors.length !== 0) {
-        let delimiter;
+            window.alert(data.errors);
-        if (csvRaw.substr(0, 1) === '"') {
-            separator = '";"';
-            delimiter = '"'
-        }
-        else {
-            separator = ';';
-            delimiter = '';
         }
 
-        // Gets first line
+        let toValidate = data.data;
-        let headersFields = lines.shift();
-        let headers;
-        if (delimiter === "") headers = headersFields.split(separator);
-        else headers = headersFields.substr(1, headersFields.length - 2).split(separator);
-        let expectedFieldCount = headers.length;
-
-        let toValidate = [];
-        let lineCounter = 1;
-        for (let line of lines) {
-
-            if (delimiter !== '') {
-                line = line.substr(1, line.length - 2);
-            }
-
-            // Remove fully empty lines (both without content and those with fields set up,
-            // but without content there.
-            if (line.length <= headers.length) continue;
-
-            let lineContents = {};
-            let fields = line.split(separator);
-
-            if (fields.length !== headers.length) {
-                this.errors.parsing.push("Number of columns in line " + lineCounter + " does not match number of headers");
-            }
-
-            for (let i = 0, max = fields.length; i < max; i++) {
-                if (headers[i] === undefined || headers[i] === null) {
-                    this.errors.parsing.push("ERROR parsing line " + lineCounter + "; column " + i);
-                    continue;
-                }
-                lineContents[headers[i]] = fields[i];
-            }
-
-            // Skip totally empty lines
-            if (Object.values(lineContents).join("").length === 0) continue;
-
-            toValidate.push(lineContents);
-            lineCounter++;
-        }
 
         this.toValidate = toValidate;
 
@@ -577,6 +531,8 @@ class CsvxmlPage {
 
         reader.onload = function() {
 
+            function handleValidation() {
+
                 // On loading success, check if the upload is valid JSON
                 console.log("Read file");
                 // Validate the file
@@ -591,6 +547,24 @@ class CsvxmlPage {
                     app.listValidationErrors(validator);
                 }
 
+            }
+
+            console.log("Postload papaparse");
+            if (typeof Papa === "undefined") {
+
+                const loadScript = document.createElement("script");
+                loadScript.setAttribute("src", "assets/js/papaparse/papaparse.min.js");
+                loadScript.addEventListener('load', function() {
+                    // console.log("Post-loaded OpenLayers");
+                    handleValidation();
+                }, {passive: true, once: true});
+                document.body.appendChild(loadScript);
+
+            }
+            else {
+                handleValidation();
+            }
+
         };
         reader.onerror = function() {
             alert(reader.error);

public/assets/js/csvxmlV2.min.js

@@ -1,12 +1,6 @@
 "use strict";if('serviceWorker'in navigator){console.log("Registering service worker");navigator.serviceWorker.register('/sw.js');}
-class CsvxmlValidator{fieldList;toValidate;errors;constructor(fieldList,csvRaw){this.errors={parsing:[],mandatoryTags:[],duplicateInvNos:[],dependentColumns:[],controlledLists:[],mainImageResource:[],};this.fieldList=Object.freeze(fieldList);const lines=csvRaw.trim().replace("\r\n","\n").split("\n");let separator;let delimiter;if(csvRaw.substr(0,1)==='"'){separator='";"';delimiter='"'}
+class CsvxmlValidator{fieldList;toValidate;errors;constructor(fieldList,csvRaw){this.errors={parsing:[],mandatoryTags:[],duplicateInvNos:[],dependentColumns:[],controlledLists:[],mainImageResource:[],};this.fieldList=Object.freeze(fieldList);const data=Papa.parse(csvRaw.trim(),{header:true});if(data.errors.length!==0){window.alert(data.errors);}
-else{separator=';';delimiter='';}
+let toValidate=data.data;this.toValidate=toValidate;if(toValidate.length===0){alert("Error: No lines of content identified");}
-let headersFields=lines.shift();let headers;if(delimiter==="")headers=headersFields.split(separator);else headers=headersFields.substr(1,headersFields.length-2).split(separator);let expectedFieldCount=headers.length;let toValidate=[];let lineCounter=1;for(let line of lines){if(delimiter!==''){line=line.substr(1,line.length-2);}
-if(line.length<=headers.length)continue;let lineContents={};let fields=line.split(separator);if(fields.length!==headers.length){this.errors.parsing.push("Number of columns in line "+lineCounter+" does not match number of headers");}
-for(let i=0,max=fields.length;i<max;i++){if(headers[i]===undefined||headers[i]===null){this.errors.parsing.push("ERROR parsing line "+lineCounter+"; column "+i);continue;}
-lineContents[headers[i]]=fields[i];}
-if(Object.values(lineContents).join("").length===0)continue;toValidate.push(lineContents);lineCounter++;}
-this.toValidate=toValidate;if(toValidate.length===0){alert("Error: No lines of content identified");}
 this.validate();}
 validate(){this.validateMandatoryTagsPresent();this.checkDuplicateInvNos();this.checkDependentColumns();this.checkControlledLists();}
 validateMandatoryTagsPresent(){let mandatoryFields=[];for(let fieldName in this.fieldList){if(this.fieldList[fieldName].required===true){mandatoryFields.push(fieldName);}}
@@ -50,8 +44,10 @@ else{runZipping();}}
 listValidationErrors(validator){console.log("Listing validation errors");const dialogueContent=document.createElement("div");const headline=document.createElement("h3");headline.textContent=this.tls.validation_errors;const cancelB=document.createElement("a");cancelB.classList.add("icons");cancelB.classList.add("iconsClose");cancelB.classList.add("dialogueCloseX");cancelB.id="dialogueClose";cancelB.textContent="X";cancelB.title="Close";cancelB.href="#"+location.href;cancelB.addEventListener('click',CsvxmlDialogue.closeDialogue);headline.appendChild(cancelB);dialogueContent.appendChild(headline);const domErrorsSection=document.createElement("div");for(let errorType in validator.errors){if(validator.errors[errorType].length===0)continue;const ulHl=document.createElement("h4");ulHl.textContent=this.tls['errors_'+errorType]+" ("+validator.errors[errorType].length+")";ulHl.style.cursor="pointer";domErrorsSection.appendChild(ulHl);const ul=document.createElement("ul");for(let error of validator.errors[errorType]){const li=document.createElement("li");li.textContent=error;ul.appendChild(li);}
 ulHl.addEventListener('click',function(){ul.classList.toggle("minimized");});domErrorsSection.appendChild(ul);}
 dialogueContent.appendChild(domErrorsSection);const domDlSection=document.createElement("div");const domDlA=document.createElement("span");domDlA.textContent=this.tls.download;domDlA.classList.add("buttonLike");let app=this;domDlA.addEventListener('click',function(){app.zipUploadToXml(validator);});domDlSection.appendChild(domDlA);dialogueContent.appendChild(domDlSection);dialogue=CsvxmlDialogue.drawDialogue(dialogueContent);}
-uploadFileForValidation(file){const reader=new FileReader();reader.readAsText(file);let app=this;document.body.classList.add("loading");reader.onload=function(){console.log("Read file");let validator=new CsvxmlValidator(app.fieldListFlat,reader.result);document.body.classList.remove("loading");if(validator.isValid()===true){alert("Document is valid. Press ok to download.");app.zipUploadToXml(validator);}
+uploadFileForValidation(file){const reader=new FileReader();reader.readAsText(file);let app=this;document.body.classList.add("loading");reader.onload=function(){function handleValidation(){console.log("Read file");let validator=new CsvxmlValidator(app.fieldListFlat,reader.result);document.body.classList.remove("loading");if(validator.isValid()===true){alert("Document is valid. Press ok to download.");app.zipUploadToXml(validator);}
-else{console.log("Identified invalid upload document");app.listValidationErrors(validator);}};reader.onerror=function(){alert(reader.error);};}
+else{console.log("Identified invalid upload document");app.listValidationErrors(validator);}}
+console.log("Postload papaparse");if(typeof Papa==="undefined"){const loadScript=document.createElement("script");loadScript.setAttribute("src","assets/js/papaparse/papaparse.min.js");loadScript.addEventListener('load',function(){handleValidation();},{passive:true,once:true});document.body.appendChild(loadScript);}
+else{handleValidation();}};reader.onerror=function(){alert(reader.error);};}
 renderGenHeader(){const header=document.createElement("header");header.id="mainHeader";const logoArea=document.createElement("a");logoArea.id="logoArea";logoArea.href="https://www.museum-digital.org/";const logoImg=document.createElement("img");logoImg.src="assets/img/mdlogo-code-128px.png";logoImg.alt="Logo of museum-digital";logoArea.appendChild(logoImg);const h2=document.createElement("h2");h2.textContent="museum-digital";logoArea.appendChild(h2);header.appendChild(logoArea);const nav=document.createElement("nav");const lAbout=document.createElement("a");lAbout.href="https://en.about.museum-digital.org/about";lAbout.textContent=this.tls.about;nav.appendChild(lAbout);const lContactList=document.createElement("div");const lContact=document.createElement("a");lContact.textContent=this.tls.contact;lContact.href="https://en.about.museum-digital.org/contact/";lContactList.appendChild(lContact);const lContactDiv=document.createElement("div");const lImprint=document.createElement("a");lImprint.textContent=this.tls.imprint;lImprint.href="https://en.about.museum-digital.org/impressum";lContactDiv.appendChild(lImprint);const lPrivacy=document.createElement("a");lPrivacy.textContent=this.tls.privacy_policy;lPrivacy.href="https://en.about.museum-digital.org/privacy/";lContactDiv.appendChild(lPrivacy);lContactList.appendChild(lContactDiv);nav.appendChild(lContactList);const lNews=document.createElement("a")
 lNews.textContent=this.tls.news;lNews.href="https://blog.museum-digital.org/";nav.appendChild(lNews);header.appendChild(nav);document.body.appendChild(header);}
 renderHeader(){const appHeader=document.createElement("header");const h1=document.createElement("h1");const img=document.createElement("img");img.width="70";img.height="70";img.src="assets/img/mdlogo-csvxml.svg";img.alt="";h1.appendChild(img);const h1Span=document.createElement("span");h1Span.textContent="museum-digital:csvxml";h1.appendChild(h1Span);appHeader.appendChild(h1);document.body.appendChild(appHeader);}

public/assets/js/papaparse/papaparse.min.js

@@ -0,0 +1,7 @@
+/* @license
+Papa Parse
+v5.3.2
+https://github.com/mholt/PapaParse
+License: MIT
+*/
+!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var r=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),b.BLOB_URL||(b.BLOB_URL=i.createObjectURL(new Blob(["(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=_,t.id=h++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);("boolean"==typeof t.escapeFormulae||t.escapeFormulae instanceof RegExp)&&(o=t.escapeFormulae instanceof RegExp?t.escapeFormulae:/^[=+\-@\t\r].*$/)}();var h=new RegExp(j(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0<e.length,s=!Array.isArray(t[0]);if(n&&_){for(var a=0;a<e.length;a++)0<a&&(r+=m),r+=v(e[a],a);0<t.length&&(r+=y)}for(var o=0;o<t.length;o++){var h=n?e.length:t[o].length,u=!1,f=n?0===Object.keys(t[o]).length:0===t[o].length;if(i&&!n&&(u="greedy"===i?""===t[o].join("").trim():1===t[o].length&&0===t[o][0].length),"greedy"===i&&n){for(var d=[],l=0;l<h;l++){var c=s?e[l]:l;d.push(t[o][c])}u=""===d.join("").trim()}if(!u){for(var p=0;p<h;p++){0<p&&!f&&(r+=m);var g=n&&s?e[p]:p;r+=v(t[o][g],p)}o<t.length-1&&(!i||0<h&&!f)&&(r+=y)}}return r}function v(e,t){if(null==e)return"";if(e.constructor===Date)return JSON.stringify(e).slice(1,25);var i=!1;o&&"string"==typeof e&&o.test(e)&&(e="'"+e,i=!0);var r=e.toString().replace(h,a);return(i=i||!0===n||"function"==typeof n&&n(e,t)||Array.isArray(n)&&n[t]||function(e,t){for(var i=0;i<t.length;i++)if(-1<e.indexOf(t[i]))return!0;return!1}(r,b.BAD_DELIMITERS)||-1<r.indexOf(m)||" "===r.charAt(0)||" "===r.charAt(r.length-1))?s+r+s:r}}};if(b.RECORD_SEP=String.fromCharCode(30),b.UNIT_SEP=String.fromCharCode(31),b.BYTE_ORDER_MARK="\ufeff",b.BAD_DELIMITERS=["\r","\n",'"',b.BYTE_ORDER_MARK],b.WORKERS_SUPPORTED=!n&&!!f.Worker,b.NODE_STREAM_INPUT=1,b.LocalChunkSize=10485760,b.RemoteChunkSize=5242880,b.DefaultDelimiter=",",b.Parser=E,b.ParserHandle=i,b.NetworkStreamer=l,b.FileStreamer=c,b.StringStreamer=p,b.ReadableStreamStreamer=g,f.jQuery){var d=f.jQuery;d.fn.parse=function(o){var i=o.config||{},h=[];return this.each(function(e){if(!("INPUT"===d(this).prop("tagName").toUpperCase()&&"file"===d(this).attr("type").toLowerCase()&&f.FileReader)||!this.files||0===this.files.length)return!0;for(var t=0;t<this.files.length;t++)h.push({file:this.files[t],inputElem:this,instanceConfig:d.extend({},i)})}),e(),this;function e(){if(0!==h.length){var e,t,i,r,n=h[0];if(M(o.before)){var s=o.before(n.file,n.inputElem);if("object"==typeof s){if("abort"===s.action)return e="AbortError",t=n.file,i=n.inputElem,r=s.reason,void(M(o.error)&&o.error({name:e},t,i,r));if("skip"===s.action)return void u();"object"==typeof s.config&&(n.instanceConfig=d.extend(n.instanceConfig,s.config))}else if(Line truncated

public/index.htm

@@ -4,7 +4,7 @@
 
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
-    <link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000002" />
+    <link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000003" />
     <link rel="manifest" href="/manifest.json" />
     <meta name="theme-color" content="#aa4400" />
 
@@ -27,7 +27,7 @@
 </head>
 <body class="loading">
 
-<script src="assets/js/csvxmlV2.min.js?v000002" type="text/javascript" async></script>
+<script src="assets/js/csvxmlV2.min.js?v000003" type="text/javascript" async></script>
 
 </body>
 </html>

public/sw.js

@@ -4,8 +4,8 @@ const urlsToCache = [
 
     '/index.htm',
     '/assets/js/jszip/dist/jszip.min.js',
-    '/assets/js/csvxmlV2.min.js',
+    '/assets/js/csvxmlV2.min.js?v000003',
-    '/assets/css/csvxml.min.css',
+    '/assets/css/csvxml.min.css?v000003',
 
     '/assets/fonts/SourceSansPro-Regular.woff2',
 

scripts/compile.php

@@ -76,7 +76,7 @@ function generateAppShell():string {
 
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
-    <link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000002" />
+    <link rel="stylesheet" type="text/css" href="assets/css/csvxml.min.css?v000003" />
     <link rel="manifest" href="/manifest.json" />
     <meta name="theme-color" content="#aa4400" />
 
@@ -99,7 +99,7 @@ function generateAppShell():string {
 </head>
 <body class="loading">
 
-<script src="assets/js/csvxmlV2.min.js?v000002" type="text/javascript" async></script>
+<script src="assets/js/csvxmlV2.min.js?v000003" type="text/javascript" async></script>
 
 </body>
 </html>';