museum-digital/museum-digital-webdav-uploader
36
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add built-in scheduler when running --webui

Browse Source 
Also:
- Default to webui

Close #16
This commit is contained in:
Joshua Ramon Enslin 2025-03-04 20:03:34 +01:00
parent 1f81d48aad
commit 0196476370
Signed by: jrenslin
GPG Key ID: 46016F84501B70AE
12 changed files with 239 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
go.mod
View File

@ -2,4 +2,12 @@ module gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader
go 1.24.0
require github.com/studio-b12/gowebdav v0.10.0 // indirect
require (
github.com/go-co-op/gocron/v2 v2.16.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
github.com/madflojo/tasks v1.2.1 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/studio-b12/gowebdav v0.10.0 // indirect
)

10
go.sum
View File

@ -1,2 +1,12 @@
github.com/go-co-op/gocron/v2 v2.16.0 h1:uqUF6WFZ4enRU45pWFNcn1xpDLc+jBOTKhPQI16Z1xs=
github.com/go-co-op/gocron/v2 v2.16.0/go.mod h1:opexeOFy5BplhsKdA7bzY9zeYih8I8/WNJ4arTIFPVc=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
github.com/madflojo/tasks v1.2.1/go.mod h1:/WMv6u3Xb5eyy+aIM76ildaIT166GOxN/jya9oI7dyo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/studio-b12/gowebdav v0.10.0 h1:Yewz8FFiadcGEu4hxS/AAJQlHelndqln1bns3hcJIYc=
github.com/studio-b12/gowebdav v0.10.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=

4
main.go
View File

@ -20,7 +20,7 @@ func main() {
} else if slices.Contains(os.Args, "--set-additional-setting") {
cli.SetAdditionalSetting()
} else if slices.Contains(os.Args, "--upload") {
cli.HandleUpload()
cli.Upload()
} else if slices.Contains(os.Args, "--webdav-remote-list-toplevel") {
cli.ListRemoteToplevel()
} else if slices.Contains(os.Args, "--webdav-remote-list-metadata-dir") {
@ -34,7 +34,7 @@ func main() {
} else if slices.Contains(os.Args, "--webui") {
webui.Run()
} else {
// cli.RunManualSetup()
webui.Run()
}
}

4
makefile
View File

@ -13,10 +13,10 @@ run:
compile:
echo "Compiling for every OS and Platform"
GOOS=linux GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-linux-amd64 main.go
GOOS=windows GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-amd64 main.go
GOOS=windows GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-amd64.exe main.go
GOOS=darwin GOARCH=amd64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-macos-amd64 main.go
GOOS=linux GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-linux-arm64 main.go
GOOS=windows GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-arm64 main.go
GOOS=windows GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-windows-arm64.exe main.go
GOOS=darwin GOARCH=arm64 go build -o bin/museum-digital-webdav-uploader-$(VERSION)-macos-arm64 main.go
all: hello build

53
src/cli/Upload.go Normal file
View File

@ -0,0 +1,53 @@
package cli
import (
"fmt"
"os"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/uploadsrcdir"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/webdavupload"
)
// Integration: Uploads, if there is uploadable data and no import currently
// scheduled.
func Upload() {
config := loadConfigFromFile()
uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
// If there are no files to upload, do nothing
if len(uploadableMetadata) == 0 && len(uploadableMedia) == 0 {
fmt.Println("No uploadable files identified.")
return
}
// Ensure no files are too new too recent - if any are, say so and abort.
if uploadsrcdir.CheckAnyFileIsTooRecent(uploadableMedia) || uploadsrcdir.CheckAnyFileIsTooRecent(uploadableMetadata) {
fmt.Println("Some of the files have been updated too recently (1min). Blocking upload to prevent accidental uploads in the middle of an ongoing digitization effort.")
return
}
// Open WebDAV client
c := webdavupload.GetWebdavClient(config)
// Check that the remote is not currently occupied.
if webdavupload.CheckRemoteIsFree(c) == false {
fmt.Println("The remote is currently occupied (very recent files, or an import config is currently waiting to be processed, are present).")
return
}
// Upload the files
if len(uploadableMetadata) > 0 {
webdavupload.UploadMetadataFiles(c, os.Stdout, uploadableMetadata)
webdavupload.BatchUnlink(os.Stdout, uploadableMetadata)
}
if len(uploadableMedia) > 0 {
webdavupload.UploadMediaFiles(c, os.Stdout, uploadableMedia)
webdavupload.BatchUnlink(os.Stdout, uploadableMedia)
}
// Generate the import config.
fmt.Println("Generating and uploading import configuration")
webdavupload.SetImportConfigToTheRemote(c, config)
fmt.Println("DONE")
}

41
src/cli/cli.go
View File

@ -372,47 +372,6 @@ func ListLocalMedia() {
}
// Integration: Uploads, if there is uploadable data and no import currently
// scheduled.
func HandleUpload() {
config := loadConfigFromFile()
uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
// If there are no files to upload, do nothing
if len(uploadableMetadata) == 0 && len(uploadableMedia) == 0 {
fmt.Println("No uploadable files identified.")
return
}
// TODO: Check, that the import files are not too new
// Open WebDAV client
c := webdavupload.GetWebdavClient(config)
// Check that the remote is not currently occupied.
if webdavupload.CheckRemoteIsFree(c) == false {
fmt.Println("The remote is currently occupied (very recent files, or an import config is currently waiting to be processed, are present).")
return
}
// Upload the files
if len(uploadableMetadata) > 0 {
webdavupload.UploadMetadataFiles(c, os.Stdout, uploadableMetadata)
webdavupload.BatchUnlink(os.Stdout, uploadableMetadata)
}
if len(uploadableMedia) > 0 {
webdavupload.UploadMediaFiles(c, os.Stdout, uploadableMedia)
webdavupload.BatchUnlink(os.Stdout, uploadableMedia)
}
// Generate the import config.
fmt.Println("Generating and uploading import configuration")
webdavupload.SetImportConfigToTheRemote(c, config)
fmt.Println("DONE")
}
// Prints current version.
func ShowVersion() {
fmt.Println(meta.GetVersion())

52
src/uploadsrcdir/ListLocalDirs.go
View File

@ -2,11 +2,14 @@ package uploadsrcdir
import (
"path/filepath"
"slices"
"os"
"io/fs"
"fmt"
"io/fs"
"os"
"runtime"
"slices"
"strings"
"sync"
"time"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
)
@ -82,3 +85,46 @@ func GetUploadableFiles(config configloader.MDWebDavUploaderConfig) ([]string, [
return metadataFiles, mediaFiles
}
// Checks if a list of files have been modified in the last minute.
// If so, returns true.
func CheckAnyFileIsTooRecent(files []string) bool {
now := time.Now()
maxConcTasks := min(10, runtime.NumCPU())
// Set a semaphore to restrict the number of concurrent upload tasks.
semaphore := make(chan struct{}, maxConcTasks)
wg := &sync.WaitGroup{}
output := false
for _, f := range(files) {
semaphore <- struct{}{} // acquire
wg.Add(1)
go func() {
defer wg.Done()
file, fStatErr := os.Stat(f)
if fStatErr != nil {
panic("Failed to read file: " + f)
}
if diff := now.Sub(file.ModTime()); diff < 1 * time.Minute {
print("File " + file.Name() + " has been changed within the last minute\n")
output = true
}
<-semaphore // release
}()
}
wg.Wait()
return output
}

4
src/webui/assets/md-uploader.css
View File

@ -150,6 +150,10 @@ iframe { display: block; width: 100%; border: none; height: 60vh; }
.parser-desc { display: block; padding: 0 .8rem;
font-size: .85em; white-space: pre-wrap; }
#wrap.setup-required { grid-template-columns: 1fr 450px 1fr; justify-content: center; }
.setup-required aside { grid-column: 2; width: 450px; margin: auto; }
.setup-required main { display: none; }
/***
* Slider
* Thanks: mrhyddenn

82
src/webui/assets/md-uploader.js
View File

@ -45,7 +45,7 @@ class App {
async generateSidebar() {
this.aside.appendChild(this.generateTextElement("h2", this.tls.settings)); // TODO
this.aside.appendChild(this.generateTextElement("h2", this.tls.settings));
// Set up fields
const instanceLinkArea = this.generateLabelInput("instance-link",
@ -304,6 +304,36 @@ class App {
}
manageUpload() {
const transferOverlay = document.createElement("div");
transferOverlay.id = "transfer-overlay";
transferOverlay.classList.add("transfer-overlay");
const hl = document.createElement("div");
hl.classList.add("transfer-overlay-hl");
hl.appendChild(this.generateTextElement("h3", this.tls.upload));
const closeB = document.createElement("span");
closeB.classList.add("overlay-close");
hl.appendChild(closeB);
closeB.addEventListener('click', function() { location.reload(); });
// closeB.addEventListener('click', function() { transferOverlay.parentElement.removeChild(transferOverlay); });
transferOverlay.appendChild(hl);
const transferMsgs = document.createElement("div");
transferMsgs.classList.add("transfer-msgs");
transferOverlay.appendChild(transferMsgs);
this.wrap.appendChild(transferOverlay);
const iframe = document.createElement("iframe");
iframe.src = "/trigger-upload";
transferOverlay.appendChild(iframe);
}
generateActionsMenu() {
if (this.uploadableFiles.metadata.length === 0 && this.uploadableFiles.media_files.length === 0) {
@ -316,35 +346,29 @@ class App {
const app = this;
uploadTrigger.addEventListener('click', function() {
const transferOverlay = document.createElement("div");
transferOverlay.id = "transfer-overlay";
transferOverlay.classList.add("transfer-overlay");
const hl = document.createElement("div");
hl.classList.add("transfer-overlay-hl");
hl.appendChild(app.generateTextElement("h3", app.tls.upload));
const closeB = document.createElement("span");
closeB.classList.add("overlay-close");
hl.appendChild(closeB);
closeB.addEventListener('click', function() { location.reload(); });
// closeB.addEventListener('click', function() { transferOverlay.parentElement.removeChild(transferOverlay); });
transferOverlay.appendChild(hl);
const transferMsgs = document.createElement("div");
transferMsgs.classList.add("transfer-msgs");
transferOverlay.appendChild(transferMsgs);
app.wrap.appendChild(transferOverlay);
const iframe = document.createElement("iframe");
iframe.src = "/trigger-upload";
transferOverlay.appendChild(iframe);
app.manageUpload();
}, {once: true});
// Handle next auto upload
const nextAutoUpload = document.createElement("span");
const startTimeStr = "Next scheduled check: ";
this.actionsArea.appendChild(nextAutoUpload);
const options1 = {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
};
const dateTimeFormat1 = new Intl.DateTimeFormat(this.lang, options1);
const nextUploadTime = Date.parse(this.config.next_auto_upload);
const now = new Date();
const diffMs = nextUploadTime - now;
const diffMins = Math.round((diffMs % 86400000) / 60000); // minutes
nextAutoUpload.textContent = dateTimeFormat1.format(nextUploadTime) + " (in " + diffMins + " mins)";
}
@ -377,7 +401,7 @@ class App {
render() {
if (this.config.setup_required === true) {
wrap.classList.add("one-column");
wrap.classList.add("setup-required");
this.generateSidebar();
}
else {

13
src/webui/serveApiGetSettings.go
View File

@ -5,18 +5,29 @@ import (
"fmt"
"net/http"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/meta"
)
type getSettingsApiResponse struct {
Version string `json:"version"`
SetupRequired bool `json:"setup_required"`
Settings configloader.MDWebDavUploaderConfig `json:"settings"`
NextAutoUpload string `json:"next_auto_upload"`
}
// Generates the API output for listing current settings.
func serveApiGetSettings(w http.ResponseWriter, r *http.Request) {
var nextAutoUpOut string
nextAutoUp, err := scheduledUpload.NextRun()
if err != nil {
nextAutoUpOut = "";
} else {
nextAutoUpOut = nextAutoUp.Format("2006-01-02 15:04:05")
}
setHeadersForJson(w)
output := getSettingsApiResponse{SetupRequired: setupRequired, Settings: config}
output := getSettingsApiResponse{SetupRequired: setupRequired, Settings: config, Version: meta.GetVersion(), NextAutoUpload: nextAutoUpOut}
outputJson, encodeErr := json.Marshal(output)
if encodeErr != nil {

10
src/webui/serveApiListUploadable.go
View File

@ -18,14 +18,14 @@ func serveApiListUploadable(w http.ResponseWriter, r *http.Request) {
setHeadersForJson(w)
var output listUploadableApiResponse
if setupRequired == true {
fmt.Fprint(w, "{'metadata': '', 'media_files': []}")
return
output = listUploadableApiResponse{Metadata: []string{}, MediaFiles: []string{}}
} else {
uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
output = listUploadableApiResponse{Metadata: uploadableMetadata, MediaFiles: uploadableMedia}
}
uploadableMetadata, uploadableMedia := uploadsrcdir.GetUploadableFiles(config)
output := listUploadableApiResponse{Metadata: uploadableMetadata, MediaFiles: uploadableMedia}
outputJson, encodeErr := json.Marshal(output)
if encodeErr != nil {
panic("Failed to create JSON")

42
src/webui/server.go
View File

@ -6,11 +6,16 @@ import (
"net/http"
"os/exec"
"runtime"
"time"
"github.com/go-co-op/gocron/v2"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/cli"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
)
var config configloader.MDWebDavUploaderConfig
var setupRequired bool
var scheduler gocron.Scheduler
var scheduledUpload gocron.Job
// Opens the application in the default browser.
func openInBrowser(port string) {
@ -33,6 +38,37 @@ func openInBrowser(port string) {
}
func scheduleUploads() {
s, err := gocron.NewScheduler()
defer func() { _ = s.Shutdown() }()
if err != nil {
panic("Failed to set up scheduler")
}
j, jobErr := s.NewJob(
gocron.DurationJob(
3 * time.Hour,
),
gocron.NewTask(
func() {
fmt.Println("Checking if an upload should be done")
fmt.Println(time.Now().Format("20060102150405"))
cli.Upload()
},
),
)
if jobErr != nil {
panic("Failed to set up scheduler job")
}
scheduledUpload = j
s.Start()
select {}
}
// Sets up the server and manages routing.
func Run() {
@ -46,7 +82,6 @@ func Run() {
http.HandleFunc("/list-parsers", serveApiListParsers) // API to list available parsers.
http.HandleFunc("/update-settings", serveApiUpdateSettings) // API for updating settings
http.HandleFunc("/trigger-upload", serveApiTriggerUpload) // Serve page for specific files
// http.HandleFunc("/contribute.json", ) // Serve page for specific files
http.HandleFunc("/md-uploader.css", serveCss) // Main CSS file.
http.HandleFunc("/md-uploader.js", serveJs) // Main JS file.
http.HandleFunc("/logo.svg", serveLogo) // Logo.
@ -63,8 +98,11 @@ func Run() {
}();
go func() {
time.Sleep(5)
openInBrowser(port)
}();
select {}
go scheduleUploads()
select{}
}