diff --git a/go.mod b/go.mod index e3e1e3b..aa063a3 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,12 @@ module gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader go 1.24.0 -require github.com/studio-b12/gowebdav v0.10.0 // indirect +require ( + github.com/go-co-op/gocron/v2 v2.16.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jonboulle/clockwork v0.5.0 // indirect + github.com/madflojo/tasks v1.2.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/studio-b12/gowebdav v0.10.0 // indirect +) diff --git a/go.sum b/go.sum index c0b6847..d66f02f 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,12 @@ +github.com/go-co-op/gocron/v2 v2.16.0 h1:uqUF6WFZ4enRU45pWFNcn1xpDLc+jBOTKhPQI16Z1xs= +github.com/go-co-op/gocron/v2 v2.16.0/go.mod h1:opexeOFy5BplhsKdA7bzY9zeYih8I8/WNJ4arTIFPVc= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/madflojo/tasks v1.2.1/go.mod h1:/WMv6u3Xb5eyy+aIM76ildaIT166GOxN/jya9oI7dyo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/studio-b12/gowebdav v0.10.0 h1:Yewz8FFiadcGEu4hxS/AAJQlHelndqln1bns3hcJIYc= github.com/studio-b12/gowebdav v0.10.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= diff --git a/main.go b/main.go index 3c1d3d4..832139a 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,7 @@ func main() { } else if slices.Contains(os.Args, "--set-additional-setting") { cli.SetAdditionalSetting() } else if slices.Contains(os.Args, "--upload") { - cli.HandleUpload() + cli.Upload() } else if slices.Contains(os.Args, "--webdav-remote-list-toplevel") { cli.ListRemoteToplevel() } else if slices.Contains(os.Args, "--webdav-remote-list-metadata-dir") { @@ -34,7 +34,7 @@ func main() { } else if slices.Contains(os.Args, "--webui") { webui.Run() } else { - // cli.RunManualSetup() + webui.Run() } } diff --git a/makefile b/makefile index d2a4c2d..c6b0792 100644 --- a/makefile +++ b/makefile @@ -13,10 +13,10 @@ run: compile: echo "Compiling for every OS and Platform" GOOS=linux GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-linux-amd64 main.go - GOOS=windows GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-amd64 main.go + GOOS=windows GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-amd64.exe main.go GOOS=darwin GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-macos-amd64 main.go GOOS=linux GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-linux-arm64 main.go - GOOS=windows GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-arm64 main.go + GOOS=windows GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-arm64.exe main.go GOOS=darwin GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-macos-arm64 main.go all: hello build diff --git a/src/cli/Upload.go b/src/cli/Upload.go new file mode 100644 index 0000000..186524f --- /dev/null +++ b/src/cli/Upload.go @@ -0,0 +1,53 @@ +package cli + +import ( + "fmt" + "os" + "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/uploadsrcdir" + "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/webdavupload" +) + +// Integration: Uploads, if there is uploadable data and no import currently +// scheduled. +func Upload() { + + config := loadConfigFromFile() + uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config) + + // If there are no files to upload, do nothing + if len(uploadableMetadata) == 0 && len(uploadableMedia) == 0 { + fmt.Println("No uploadable files identified.") + return + } + + // Ensure no files are too new too recent - if any are, say so and abort. + if uploadsrcdir.CheckAnyFileIsTooRecent(uploadableMedia) || uploadsrcdir.CheckAnyFileIsTooRecent(uploadableMetadata) { + fmt.Println("Some of the files have been updated too recently (1min). Blocking upload to prevent accidental uploads in the middle of an ongoing digitization effort.") + return + } + + // Open WebDAV client + c := webdavupload.GetWebdavClient(config) + + // Check that the remote is not currently occupied. + if webdavupload.CheckRemoteIsFree(c) == false { + fmt.Println("The remote is currently occupied (very recent files, or an import config is currently waiting to be processed, are present).") + return + } + + // Upload the files + if len(uploadableMetadata) > 0 { + webdavupload.UploadMetadataFiles(c, os.Stdout, uploadableMetadata) + webdavupload.BatchUnlink(os.Stdout, uploadableMetadata) + } + if len(uploadableMedia) > 0 { + webdavupload.UploadMediaFiles(c, os.Stdout, uploadableMedia) + webdavupload.BatchUnlink(os.Stdout, uploadableMedia) + } + + // Generate the import config. + fmt.Println("Generating and uploading import configuration") + webdavupload.SetImportConfigToTheRemote(c, config) + fmt.Println("DONE") + +} diff --git a/src/cli/cli.go b/src/cli/cli.go index e1ac8f2..f24dc53 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -372,47 +372,6 @@ func ListLocalMedia() { } -// Integration: Uploads, if there is uploadable data and no import currently -// scheduled. -func HandleUpload() { - - config := loadConfigFromFile() - uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config) - - // If there are no files to upload, do nothing - if len(uploadableMetadata) == 0 && len(uploadableMedia) == 0 { - fmt.Println("No uploadable files identified.") - return - } - - // TODO: Check, that the import files are not too new - - // Open WebDAV client - c := webdavupload.GetWebdavClient(config) - - // Check that the remote is not currently occupied. - if webdavupload.CheckRemoteIsFree(c) == false { - fmt.Println("The remote is currently occupied (very recent files, or an import config is currently waiting to be processed, are present).") - return - } - - // Upload the files - if len(uploadableMetadata) > 0 { - webdavupload.UploadMetadataFiles(c, os.Stdout, uploadableMetadata) - webdavupload.BatchUnlink(os.Stdout, uploadableMetadata) - } - if len(uploadableMedia) > 0 { - webdavupload.UploadMediaFiles(c, os.Stdout, uploadableMedia) - webdavupload.BatchUnlink(os.Stdout, uploadableMedia) - } - - // Generate the import config. - fmt.Println("Generating and uploading import configuration") - webdavupload.SetImportConfigToTheRemote(c, config) - fmt.Println("DONE") - -} - // Prints current version. func ShowVersion() { fmt.Println(meta.GetVersion()) diff --git a/src/uploadsrcdir/ListLocalDirs.go b/src/uploadsrcdir/ListLocalDirs.go index 41c8c2e..07e736b 100644 --- a/src/uploadsrcdir/ListLocalDirs.go +++ b/src/uploadsrcdir/ListLocalDirs.go @@ -2,11 +2,14 @@ package uploadsrcdir import ( "path/filepath" - "slices" - "os" - "io/fs" "fmt" + "io/fs" + "os" + "runtime" + "slices" "strings" + "sync" + "time" "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader" ) @@ -82,3 +85,46 @@ func GetUploadableFiles(config configloader.MDWebDavUploaderConfig) ([]string, [ return metadataFiles, mediaFiles } + +// Checks if a list of files have been modified in the last minute. +// If so, returns true. +func CheckAnyFileIsTooRecent(files []string) bool { + + now := time.Now() + maxConcTasks := min(10, runtime.NumCPU()) + + // Set a semaphore to restrict the number of concurrent upload tasks. + semaphore := make(chan struct{}, maxConcTasks) + wg := &sync.WaitGroup{} + + output := false + + for _, f := range(files) { + + semaphore <- struct{}{} // acquire + wg.Add(1) + + go func() { + + defer wg.Done() + file, fStatErr := os.Stat(f) + if fStatErr != nil { + panic("Failed to read file: " + f) + } + + if diff := now.Sub(file.ModTime()); diff < 1 * time.Minute { + print("File " + file.Name() + " has been changed within the last minute\n") + output = true + } + + <-semaphore // release + + }() + + } + + wg.Wait() + + return output + +} diff --git a/src/webui/assets/md-uploader.css b/src/webui/assets/md-uploader.css index 3f7a5ee..ee7fd1a 100644 --- a/src/webui/assets/md-uploader.css +++ b/src/webui/assets/md-uploader.css @@ -150,6 +150,10 @@ iframe { display: block; width: 100%; border: none; height: 60vh; } .parser-desc { display: block; padding: 0 .8rem; font-size: .85em; white-space: pre-wrap; } +#wrap.setup-required { grid-template-columns: 1fr 450px 1fr; justify-content: center; } +.setup-required aside { grid-column: 2; width: 450px; margin: auto; } +.setup-required main { display: none; } + /*** * Slider * Thanks: mrhyddenn diff --git a/src/webui/assets/md-uploader.js b/src/webui/assets/md-uploader.js index d8ec231..6e6a8b8 100644 --- a/src/webui/assets/md-uploader.js +++ b/src/webui/assets/md-uploader.js @@ -45,7 +45,7 @@ class App { async generateSidebar() { - this.aside.appendChild(this.generateTextElement("h2", this.tls.settings)); // TODO + this.aside.appendChild(this.generateTextElement("h2", this.tls.settings)); // Set up fields const instanceLinkArea = this.generateLabelInput("instance-link", @@ -304,6 +304,36 @@ class App { } + manageUpload() { + + const transferOverlay = document.createElement("div"); + transferOverlay.id = "transfer-overlay"; + transferOverlay.classList.add("transfer-overlay"); + const hl = document.createElement("div"); + hl.classList.add("transfer-overlay-hl"); + hl.appendChild(this.generateTextElement("h3", this.tls.upload)); + + const closeB = document.createElement("span"); + closeB.classList.add("overlay-close"); + hl.appendChild(closeB); + + closeB.addEventListener('click', function() { location.reload(); }); + // closeB.addEventListener('click', function() { transferOverlay.parentElement.removeChild(transferOverlay); }); + + transferOverlay.appendChild(hl); + + const transferMsgs = document.createElement("div"); + transferMsgs.classList.add("transfer-msgs"); + transferOverlay.appendChild(transferMsgs); + + this.wrap.appendChild(transferOverlay); + + const iframe = document.createElement("iframe"); + iframe.src = "/trigger-upload"; + transferOverlay.appendChild(iframe); + + } + generateActionsMenu() { if (this.uploadableFiles.metadata.length === 0 && this.uploadableFiles.media_files.length === 0) { @@ -316,35 +346,29 @@ class App { const app = this; uploadTrigger.addEventListener('click', function() { - - const transferOverlay = document.createElement("div"); - transferOverlay.id = "transfer-overlay"; - transferOverlay.classList.add("transfer-overlay"); - const hl = document.createElement("div"); - hl.classList.add("transfer-overlay-hl"); - hl.appendChild(app.generateTextElement("h3", app.tls.upload)); - - const closeB = document.createElement("span"); - closeB.classList.add("overlay-close"); - hl.appendChild(closeB); - - closeB.addEventListener('click', function() { location.reload(); }); - // closeB.addEventListener('click', function() { transferOverlay.parentElement.removeChild(transferOverlay); }); - - transferOverlay.appendChild(hl); - - const transferMsgs = document.createElement("div"); - transferMsgs.classList.add("transfer-msgs"); - transferOverlay.appendChild(transferMsgs); - - app.wrap.appendChild(transferOverlay); - - const iframe = document.createElement("iframe"); - iframe.src = "/trigger-upload"; - transferOverlay.appendChild(iframe); - + app.manageUpload(); }, {once: true}); + // Handle next auto upload + const nextAutoUpload = document.createElement("span"); + const startTimeStr = "Next scheduled check: "; + this.actionsArea.appendChild(nextAutoUpload); + + const options1 = { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }; + + const dateTimeFormat1 = new Intl.DateTimeFormat(this.lang, options1); + const nextUploadTime = Date.parse(this.config.next_auto_upload); + const now = new Date(); + const diffMs = nextUploadTime - now; + + const diffMins = Math.round((diffMs % 86400000) / 60000); // minutes + nextAutoUpload.textContent = dateTimeFormat1.format(nextUploadTime) + " (in " + diffMins + " mins)"; } @@ -377,7 +401,7 @@ class App { render() { if (this.config.setup_required === true) { - wrap.classList.add("one-column"); + wrap.classList.add("setup-required"); this.generateSidebar(); } else { diff --git a/src/webui/serveApiGetSettings.go b/src/webui/serveApiGetSettings.go index b1be2ac..c3309a8 100644 --- a/src/webui/serveApiGetSettings.go +++ b/src/webui/serveApiGetSettings.go @@ -5,18 +5,29 @@ import ( "fmt" "net/http" "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader" + "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/meta" ) type getSettingsApiResponse struct { + Version string `json:"version"` SetupRequired bool `json:"setup_required"` Settings configloader.MDWebDavUploaderConfig `json:"settings"` + NextAutoUpload string `json:"next_auto_upload"` } // Generates the API output for listing current settings. func serveApiGetSettings(w http.ResponseWriter, r *http.Request) { + var nextAutoUpOut string + nextAutoUp, err := scheduledUpload.NextRun() + if err != nil { + nextAutoUpOut = ""; + } else { + nextAutoUpOut = nextAutoUp.Format("2006-01-02 15:04:05") + } + setHeadersForJson(w) - output := getSettingsApiResponse{SetupRequired: setupRequired, Settings: config} + output := getSettingsApiResponse{SetupRequired: setupRequired, Settings: config, Version: meta.GetVersion(), NextAutoUpload: nextAutoUpOut} outputJson, encodeErr := json.Marshal(output) if encodeErr != nil { diff --git a/src/webui/serveApiListUploadable.go b/src/webui/serveApiListUploadable.go index 7797945..f17f53f 100644 --- a/src/webui/serveApiListUploadable.go +++ b/src/webui/serveApiListUploadable.go @@ -18,14 +18,14 @@ func serveApiListUploadable(w http.ResponseWriter, r *http.Request) { setHeadersForJson(w) + var output listUploadableApiResponse if setupRequired == true { - fmt.Fprint(w, "{'metadata': '', 'media_files': []}") - return + output = listUploadableApiResponse{Metadata: []string{}, MediaFiles: []string{}} + } else { + uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config) + output = listUploadableApiResponse{Metadata: uploadableMetadata, MediaFiles: uploadableMedia} } - uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config) - output := listUploadableApiResponse{Metadata: uploadableMetadata, MediaFiles: uploadableMedia} - outputJson, encodeErr := json.Marshal(output) if encodeErr != nil { panic("Failed to create JSON") diff --git a/src/webui/server.go b/src/webui/server.go index d90a0ef..278a320 100644 --- a/src/webui/server.go +++ b/src/webui/server.go @@ -6,11 +6,16 @@ import ( "net/http" "os/exec" "runtime" + "time" + "github.com/go-co-op/gocron/v2" + "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/cli" "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader" ) var config configloader.MDWebDavUploaderConfig var setupRequired bool +var scheduler gocron.Scheduler +var scheduledUpload gocron.Job // Opens the application in the default browser. func openInBrowser(port string) { @@ -33,6 +38,37 @@ func openInBrowser(port string) { } +func scheduleUploads() { + + s, err := gocron.NewScheduler() + defer func() { _ = s.Shutdown() }() + + if err != nil { + panic("Failed to set up scheduler") + } + + j, jobErr := s.NewJob( + gocron.DurationJob( + 3 * time.Hour, + ), + gocron.NewTask( + func() { + fmt.Println("Checking if an upload should be done") + fmt.Println(time.Now().Format("20060102150405")) + cli.Upload() + }, + ), + ) + if jobErr != nil { + panic("Failed to set up scheduler job") + } + + scheduledUpload = j + s.Start() + select {} + +} + // Sets up the server and manages routing. func Run() { @@ -46,7 +82,6 @@ func Run() { http.HandleFunc("/list-parsers", serveApiListParsers) // API to list available parsers. http.HandleFunc("/update-settings", serveApiUpdateSettings) // API for updating settings http.HandleFunc("/trigger-upload", serveApiTriggerUpload) // Serve page for specific files - // http.HandleFunc("/contribute.json", ) // Serve page for specific files http.HandleFunc("/md-uploader.css", serveCss) // Main CSS file. http.HandleFunc("/md-uploader.js", serveJs) // Main JS file. http.HandleFunc("/logo.svg", serveLogo) // Logo. @@ -63,8 +98,11 @@ func Run() { }(); go func() { + time.Sleep(5) openInBrowser(port) }(); - select {} + go scheduleUploads() + + select{} }