Add built-in scheduler when running --webui

Also:
- Default to webui

Close #16

go.mod

@@ -2,4 +2,12 @@ module gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader
 
 go 1.24.0
 
-require github.com/studio-b12/gowebdav v0.10.0 // indirect
+require (
+	github.com/go-co-op/gocron/v2 v2.16.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/jonboulle/clockwork v0.5.0 // indirect
+	github.com/madflojo/tasks v1.2.1 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+	github.com/studio-b12/gowebdav v0.10.0 // indirect
+)

go.sum

@@ -1,2 +1,12 @@
+github.com/go-co-op/gocron/v2 v2.16.0 h1:uqUF6WFZ4enRU45pWFNcn1xpDLc+jBOTKhPQI16Z1xs=
+github.com/go-co-op/gocron/v2 v2.16.0/go.mod h1:opexeOFy5BplhsKdA7bzY9zeYih8I8/WNJ4arTIFPVc=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
+github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
+github.com/madflojo/tasks v1.2.1/go.mod h1:/WMv6u3Xb5eyy+aIM76ildaIT166GOxN/jya9oI7dyo=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/studio-b12/gowebdav v0.10.0 h1:Yewz8FFiadcGEu4hxS/AAJQlHelndqln1bns3hcJIYc=
 github.com/studio-b12/gowebdav v0.10.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=

main.go

@@ -20,7 +20,7 @@ func main() {
     } else if slices.Contains(os.Args, "--set-additional-setting") {
         cli.SetAdditionalSetting()
     } else if slices.Contains(os.Args, "--upload") {
-        cli.HandleUpload()
+        cli.Upload()
     } else if slices.Contains(os.Args, "--webdav-remote-list-toplevel") {
         cli.ListRemoteToplevel()
     } else if slices.Contains(os.Args, "--webdav-remote-list-metadata-dir") {
@@ -34,7 +34,7 @@ func main() {
     } else if slices.Contains(os.Args, "--webui") {
         webui.Run()
     } else {
-        // cli.RunManualSetup()
+        webui.Run()
     }
 
 }

makefile

@@ -13,10 +13,10 @@ run:
 compile:
 	echo "Compiling for every OS and Platform"
 	GOOS=linux   GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-linux-amd64 main.go
-	GOOS=windows GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-amd64 main.go
+	GOOS=windows GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-amd64.exe main.go
 	GOOS=darwin  GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-macos-amd64 main.go
 	GOOS=linux   GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-linux-arm64 main.go
-	GOOS=windows GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-arm64 main.go
+	GOOS=windows GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-arm64.exe main.go
 	GOOS=darwin  GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-macos-arm64 main.go
 
 all: hello build

src/cli/Upload.go

@@ -0,0 +1,53 @@
+package cli
+
+import (
+    "fmt"
+    "os"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/uploadsrcdir"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/webdavupload"
+)
+
+// Integration: Uploads, if there is uploadable data and no import currently
+// scheduled.
+func Upload() {
+
+    config := loadConfigFromFile()
+    uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
+
+    // If there are no files to upload, do nothing
+    if len(uploadableMetadata) == 0 && len(uploadableMedia) == 0 {
+        fmt.Println("No uploadable files identified.")
+        return
+    }
+
+    // Ensure no files are too new too recent - if any are, say so and abort.
+    if uploadsrcdir.CheckAnyFileIsTooRecent(uploadableMedia) || uploadsrcdir.CheckAnyFileIsTooRecent(uploadableMetadata) {
+        fmt.Println("Some of the files have been updated too recently (1min). Blocking upload to prevent accidental uploads in the middle of an ongoing digitization effort.")
+        return
+    }
+
+    // Open WebDAV client
+    c := webdavupload.GetWebdavClient(config)
+
+    // Check that the remote is not currently occupied.
+    if webdavupload.CheckRemoteIsFree(c) == false {
+        fmt.Println("The remote is currently occupied (very recent files, or an import config is currently waiting to be processed, are present).")
+        return
+    }
+
+    // Upload the files
+    if len(uploadableMetadata) > 0 {
+        webdavupload.UploadMetadataFiles(c, os.Stdout, uploadableMetadata)
+        webdavupload.BatchUnlink(os.Stdout, uploadableMetadata)
+    }
+    if len(uploadableMedia) > 0 {
+        webdavupload.UploadMediaFiles(c, os.Stdout, uploadableMedia)
+        webdavupload.BatchUnlink(os.Stdout, uploadableMedia)
+    }
+
+    // Generate the import config.
+    fmt.Println("Generating and uploading import configuration")
+    webdavupload.SetImportConfigToTheRemote(c, config)
+    fmt.Println("DONE")
+
+}

src/cli/cli.go

@@ -372,47 +372,6 @@ func ListLocalMedia() {
 
 }
 
-// Integration: Uploads, if there is uploadable data and no import currently
-// scheduled.
-func HandleUpload() {
-
-    config := loadConfigFromFile()
-    uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
-
-    // If there are no files to upload, do nothing
-    if len(uploadableMetadata) == 0 && len(uploadableMedia) == 0 {
-        fmt.Println("No uploadable files identified.")
-        return
-    }
-
-    // TODO: Check, that the import files are not too new
-
-    // Open WebDAV client
-    c := webdavupload.GetWebdavClient(config)
-
-    // Check that the remote is not currently occupied.
-    if webdavupload.CheckRemoteIsFree(c) == false {
-        fmt.Println("The remote is currently occupied (very recent files, or an import config is currently waiting to be processed, are present).")
-        return
-    }
-
-    // Upload the files
-    if len(uploadableMetadata) > 0 {
-        webdavupload.UploadMetadataFiles(c, os.Stdout, uploadableMetadata)
-        webdavupload.BatchUnlink(os.Stdout, uploadableMetadata)
-    }
-    if len(uploadableMedia) > 0 {
-        webdavupload.UploadMediaFiles(c, os.Stdout, uploadableMedia)
-        webdavupload.BatchUnlink(os.Stdout, uploadableMedia)
-    }
-
-    // Generate the import config.
-    fmt.Println("Generating and uploading import configuration")
-    webdavupload.SetImportConfigToTheRemote(c, config)
-    fmt.Println("DONE")
-
-}
-
 // Prints current version.
 func ShowVersion() {
     fmt.Println(meta.GetVersion())

src/uploadsrcdir/ListLocalDirs.go

@@ -2,11 +2,14 @@ package uploadsrcdir
 
 import (
     "path/filepath"
-    "slices"
-    "os"
-    "io/fs"
     "fmt"
+    "io/fs"
+    "os"
+    "runtime"
+    "slices"
     "strings"
+    "sync"
+    "time"
     "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
 )
 
@@ -82,3 +85,46 @@ func GetUploadableFiles(config configloader.MDWebDavUploaderConfig) ([]string, [
     return metadataFiles, mediaFiles
 
 }
+
+// Checks if a list of files have been modified in the last minute.
+// If so, returns true.
+func CheckAnyFileIsTooRecent(files []string) bool {
+
+    now := time.Now()
+    maxConcTasks := min(10, runtime.NumCPU())
+
+    // Set a semaphore to restrict the number of concurrent upload tasks.
+    semaphore := make(chan struct{}, maxConcTasks)
+    wg := &sync.WaitGroup{}
+
+    output := false
+
+    for _, f := range(files) {
+
+        semaphore <- struct{}{} // acquire
+        wg.Add(1)
+
+        go func() {
+
+            defer wg.Done()
+            file, fStatErr := os.Stat(f)
+            if fStatErr != nil {
+                panic("Failed to read file: " + f)
+            }
+
+            if diff := now.Sub(file.ModTime()); diff < 1 * time.Minute {
+                print("File " + file.Name() + " has been changed within the last minute\n")
+                output = true
+            }
+
+            <-semaphore // release
+
+        }()
+
+    }
+
+    wg.Wait()
+
+    return output
+
+}

src/webui/assets/md-uploader.css

@@ -150,6 +150,10 @@ iframe { display: block; width: 100%; border: none; height: 60vh; }
 .parser-desc { display: block; padding: 0 .8rem;
     font-size: .85em; white-space: pre-wrap; }
 
+#wrap.setup-required { grid-template-columns: 1fr 450px 1fr; justify-content: center; }
+.setup-required aside { grid-column: 2; width: 450px; margin: auto; }
+.setup-required main { display: none; }
+
 /***
  * Slider
  * Thanks: mrhyddenn

src/webui/assets/md-uploader.js

@@ -45,7 +45,7 @@ class App {
 
     async generateSidebar() {
 
-        this.aside.appendChild(this.generateTextElement("h2", this.tls.settings)); // TODO
+        this.aside.appendChild(this.generateTextElement("h2", this.tls.settings));
 
         // Set up fields
         const instanceLinkArea = this.generateLabelInput("instance-link",
@@ -304,6 +304,36 @@ class App {
 
     }
 
+    manageUpload() {
+
+        const transferOverlay = document.createElement("div");
+        transferOverlay.id = "transfer-overlay";
+        transferOverlay.classList.add("transfer-overlay");
+        const hl = document.createElement("div");
+        hl.classList.add("transfer-overlay-hl");
+        hl.appendChild(this.generateTextElement("h3", this.tls.upload));
+
+        const closeB = document.createElement("span");
+        closeB.classList.add("overlay-close");
+        hl.appendChild(closeB);
+
+        closeB.addEventListener('click', function() { location.reload(); });
+        // closeB.addEventListener('click', function() { transferOverlay.parentElement.removeChild(transferOverlay); });
+
+        transferOverlay.appendChild(hl);
+
+        const transferMsgs = document.createElement("div");
+        transferMsgs.classList.add("transfer-msgs");
+        transferOverlay.appendChild(transferMsgs);
+
+        this.wrap.appendChild(transferOverlay);
+
+        const iframe = document.createElement("iframe");
+        iframe.src = "/trigger-upload";
+        transferOverlay.appendChild(iframe);
+
+    }
+
     generateActionsMenu() {
 
         if (this.uploadableFiles.metadata.length === 0 && this.uploadableFiles.media_files.length === 0) {
@@ -316,35 +346,29 @@ class App {
 
         const app = this;
         uploadTrigger.addEventListener('click', function() {
-
+            app.manageUpload();
-            const transferOverlay = document.createElement("div");
-            transferOverlay.id = "transfer-overlay";
-            transferOverlay.classList.add("transfer-overlay");
-            const hl = document.createElement("div");
-            hl.classList.add("transfer-overlay-hl");
-            hl.appendChild(app.generateTextElement("h3", app.tls.upload));
-
-            const closeB = document.createElement("span");
-            closeB.classList.add("overlay-close");
-            hl.appendChild(closeB);
-
-            closeB.addEventListener('click', function() { location.reload(); });
-            // closeB.addEventListener('click', function() { transferOverlay.parentElement.removeChild(transferOverlay); });
-
-            transferOverlay.appendChild(hl);
-
-            const transferMsgs = document.createElement("div");
-            transferMsgs.classList.add("transfer-msgs");
-            transferOverlay.appendChild(transferMsgs);
-
-            app.wrap.appendChild(transferOverlay);
-
-            const iframe = document.createElement("iframe");
-            iframe.src = "/trigger-upload";
-            transferOverlay.appendChild(iframe);
-
         }, {once: true});
 
+        // Handle next auto upload
+        const nextAutoUpload = document.createElement("span");
+        const startTimeStr = "Next scheduled check: ";
+        this.actionsArea.appendChild(nextAutoUpload);
+
+        const options1 = {
+            year:   'numeric',
+            month:  '2-digit',
+            day:    '2-digit',
+            hour:   '2-digit',
+            minute: '2-digit',
+        };
+
+        const dateTimeFormat1 = new Intl.DateTimeFormat(this.lang, options1);
+        const nextUploadTime = Date.parse(this.config.next_auto_upload);
+        const now = new Date();
+        const diffMs = nextUploadTime - now;
+
+        const diffMins = Math.round((diffMs % 86400000) / 60000); // minutes
+        nextAutoUpload.textContent = dateTimeFormat1.format(nextUploadTime) + " (in " + diffMins + " mins)";
 
     }
 
@@ -377,7 +401,7 @@ class App {
     render() {
 
         if (this.config.setup_required === true) {
-            wrap.classList.add("one-column");
+            wrap.classList.add("setup-required");
             this.generateSidebar();
         }
         else {

src/webui/serveApiGetSettings.go

@@ -5,18 +5,29 @@ import (
 	"fmt"
 	"net/http"
     "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/meta"
 )
 
 type getSettingsApiResponse struct {
+    Version string `json:"version"`
     SetupRequired bool `json:"setup_required"`
     Settings configloader.MDWebDavUploaderConfig `json:"settings"`
+    NextAutoUpload string `json:"next_auto_upload"`
 }
 
 // Generates the API output for listing current settings.
 func serveApiGetSettings(w http.ResponseWriter, r *http.Request) {
 
+    var nextAutoUpOut string
+    nextAutoUp, err := scheduledUpload.NextRun()
+    if err != nil {
+        nextAutoUpOut = "";
+    } else {
+        nextAutoUpOut = nextAutoUp.Format("2006-01-02 15:04:05")
+    }
+
 	setHeadersForJson(w)
-    output := getSettingsApiResponse{SetupRequired: setupRequired, Settings: config}
+    output := getSettingsApiResponse{SetupRequired: setupRequired, Settings: config, Version: meta.GetVersion(), NextAutoUpload: nextAutoUpOut}
 
     outputJson, encodeErr := json.Marshal(output)
     if encodeErr != nil {

src/webui/serveApiListUploadable.go

@@ -18,14 +18,14 @@ func serveApiListUploadable(w http.ResponseWriter, r *http.Request) {
 
 	setHeadersForJson(w)
 
+    var output listUploadableApiResponse
     if setupRequired == true {
-        fmt.Fprint(w, "{'metadata': '', 'media_files': []}")
+        output = listUploadableApiResponse{Metadata: []string{}, MediaFiles: []string{}}
-        return
+    } else {
+        uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
+        output = listUploadableApiResponse{Metadata: uploadableMetadata, MediaFiles: uploadableMedia}
     }
 
-    uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
-    output := listUploadableApiResponse{Metadata: uploadableMetadata, MediaFiles: uploadableMedia}
-
     outputJson, encodeErr := json.Marshal(output)
     if encodeErr != nil {
         panic("Failed to create JSON")

src/webui/server.go

@@ -6,11 +6,16 @@ import (
 	"net/http"
     "os/exec"
     "runtime"
+    "time"
+    "github.com/go-co-op/gocron/v2"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/cli"
     "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
 )
 
 var config configloader.MDWebDavUploaderConfig
 var setupRequired bool
+var scheduler       gocron.Scheduler
+var scheduledUpload gocron.Job
 
 // Opens the application in the default browser.
 func openInBrowser(port string) {
@@ -33,6 +38,37 @@ func openInBrowser(port string) {
 
 }
 
+func scheduleUploads() {
+
+    s, err := gocron.NewScheduler()
+    defer func() { _ = s.Shutdown() }()
+
+    if err != nil {
+        panic("Failed to set up scheduler")
+    }
+
+    j, jobErr := s.NewJob(
+        gocron.DurationJob(
+            3 * time.Hour,
+        ),
+        gocron.NewTask(
+            func() {
+                fmt.Println("Checking if an upload should be done")
+                fmt.Println(time.Now().Format("20060102150405"))
+                cli.Upload()
+            },
+        ),
+    )
+    if jobErr != nil {
+        panic("Failed to set up scheduler job")
+    }
+
+    scheduledUpload = j
+    s.Start()
+    select {}
+
+}
+
 // Sets up the server and manages routing.
 func Run() {
 
@@ -46,7 +82,6 @@ func Run() {
 	http.HandleFunc("/list-parsers", serveApiListParsers)       // API to list available parsers.
     http.HandleFunc("/update-settings", serveApiUpdateSettings) // API for updating settings
 	http.HandleFunc("/trigger-upload", serveApiTriggerUpload)           // Serve page for specific files
-	// http.HandleFunc("/contribute.json", )                // Serve page for specific files
 	http.HandleFunc("/md-uploader.css", serveCss)        // Main CSS file.
 	http.HandleFunc("/md-uploader.js", serveJs)          // Main JS file.
 	http.HandleFunc("/logo.svg", serveLogo)              // Logo.
@@ -63,8 +98,11 @@ func Run() {
 
     }();
     go func() {
+        time.Sleep(5)
         openInBrowser(port)
     }();
-    select {}
+    go scheduleUploads()
+
+    select{}
 
 }