First working version

- Move from separate dirs for upload to a unified one (identify
  media/metadata files by file extension)
- Prevent uploading when an import is already scheduled
- Allow setting custom, parser-specific settings
- Add CLI
- Implement WebDAV upload
- Implement checking of upload folders for uploadable contents

Close #6, close #7, close #9, close #3, close #1, close #4

src/webdavupload/CheckRemoteContents.go

@@ -0,0 +1,115 @@
+package webdavupload
+
+import (
+    "os"
+    "sync"
+    "time"
+    "github.com/studio-b12/gowebdav"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
+)
+
+// Takes a path on the webdav remote and reads folder contents from it.
+func listWebDavFolderContents(config configloader.MDWebDavUploaderConfig, path string) ([]os.FileInfo, error) {
+
+    c := gowebdav.NewClient(getWebDavHost(config), config.Username, config.WebDavAuthToken)
+    c.Connect()
+
+    files, err := c.ReadDir(path)
+    return files, err
+
+}
+
+// Returns a list of the files on the top level of the remote.
+func ListTopLevelContents(config configloader.MDWebDavUploaderConfig) ([]os.FileInfo, error) {
+    return listWebDavFolderContents(config, ".")
+}
+
+// Returns a list of the files in the current metadata dir of the remote.
+func ListMetadataDir(config configloader.MDWebDavUploaderConfig) ([]os.FileInfo, error) {
+    return listWebDavFolderContents(config, "./IMPORT_XML")
+}
+
+// Returns a list of the files in the current media dir of the remote.
+func ListMediaDir(config configloader.MDWebDavUploaderConfig) ([]os.FileInfo, error) {
+    return listWebDavFolderContents(config, "./IMPORT_IMG")
+}
+
+func checkImportConfigExists(c *gowebdav.Client) bool {
+
+    _, err := c.Stat("./import_config.txt")
+    if gowebdav.IsErrNotFound(err) == true {
+        return false
+    }
+    return true
+
+}
+
+// Checks if a remote directory is free for updating:
+// This means it can be read, and it does not contain files more recent than
+// 10 minutes.
+func checkRemoteDirIsFree(c *gowebdav.Client, path string) bool {
+
+    now := time.Now()
+
+    // Check metadata dir
+    files, err := c.ReadDir(path)
+
+    if err != nil {
+        print("Failed to load remote contents")
+        return false
+    }
+
+    for _, f := range(files) {
+        if diff := now.Sub(f.ModTime()); diff < 10 * time.Minute {
+            print("File " + f.Name() + " has been recently renamed")
+            return false
+        }
+    }
+
+    return true
+
+}
+
+// Checks if either the metadata or the image folder contains files that have
+// been changed in the last 10 minutes.
+// In this case, it can be assumed that there is a concurrent upload taking place.
+func checkRemoteImportFilesAreTooRecent(c *gowebdav.Client) bool {
+
+    var wg sync.WaitGroup
+    var metadataDirFree bool
+    var mediaDirFree bool
+
+    wg.Add(2)
+    go func() {
+        defer wg.Done()
+        metadataDirFree = checkRemoteDirIsFree(c, "./IMPORT_XML")
+    }()
+    go func() {
+        defer wg.Done()
+        mediaDirFree = checkRemoteDirIsFree(c, "./IMPORT_IMG")
+    }()
+    wg.Wait()
+
+    if metadataDirFree == true && mediaDirFree == true {
+        return false
+    }
+    return true
+
+}
+
+// Checks the remote for the existence of an import_config.txt.
+func CheckRemoteIsFree(c *gowebdav.Client) bool {
+
+    if checkImportConfigExists(c) == true {
+        return false
+    }
+
+    // Check if the remote media and metadata directories currently
+    // contain very recent files indicating a concurrent upload.
+    if checkRemoteImportFilesAreTooRecent(c) == true {
+        return false
+    }
+
+    return true
+
+}

src/webdavupload/Client.go

@@ -0,0 +1,35 @@
+package webdavupload
+
+import (
+    "net/url"
+    "strconv"
+    "github.com/studio-b12/gowebdav"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
+)
+
+// Generates the path for a webdav endpoint for the configured instance and institution.
+func getWebDavHost(config configloader.MDWebDavUploaderConfig) string {
+
+    parsed, err := url.Parse(config.InstanceLink)
+    if err != nil {
+        panic("Your settings must be broken. Failed to load instance URL.")
+    }
+
+    output := url.URL{
+        Scheme: "https",
+        Host: parsed.Host,
+        Path: "/musdb/webdav/" + strconv.Itoa(config.InstitutionId),
+    }
+
+    return output.String()
+
+}
+
+// Returns a WebDAV client for the current config.
+func GetWebdavClient(config configloader.MDWebDavUploaderConfig) *gowebdav.Client {
+
+    c := gowebdav.NewClient(getWebDavHost(config), config.Username, config.WebDavAuthToken)
+    c.Connect()
+    return c
+
+}

src/webdavupload/Upload.go

@@ -0,0 +1,84 @@
+package webdavupload
+
+import (
+    "fmt"
+    "path/filepath"
+    "os"
+    "runtime"
+    "sync"
+    "sync/atomic"
+    "github.com/studio-b12/gowebdav"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
+    "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/importconfiggen"
+)
+
+// Writes an import config to the remote
+func SetImportConfigToTheRemote(c *gowebdav.Client, config configloader.MDWebDavUploaderConfig) error {
+
+    importConf := importconfiggen.GenerateImportConfig(config)
+    return c.Write("import_config.txt", []byte(importConf), 0660)
+
+}
+
+// Uploads a list of files to the target folder.
+func uploadFiles(c *gowebdav.Client, files []string, remoteTarget string, outputContext string) {
+
+    total := len(files)
+    var counter atomic.Uint64
+
+    // Determine the number of upload tasks to be processed concurrently.
+    // 10 will be a hard maximum to not spam the server.
+    maxConcTasks := min(10, runtime.NumCPU())
+
+    // Set a semaphore to restrict the number of concurrent upload tasks.
+    semaphore := make(chan struct{}, maxConcTasks)
+    wg := &sync.WaitGroup{}
+
+    fmt.Printf("Will upload %v files. Processing %v tasks at a time.\n", total, maxConcTasks)
+
+    for _, f := range(files) {
+
+        semaphore <- struct{}{} // acquire
+        wg.Add(1)
+
+        go func() {
+
+            defer wg.Done()
+
+            basename := filepath.Base(f)
+
+            file, fOpenErr := os.Open(f)
+            if fOpenErr != nil {
+                panic("Failed to read file: " + f)
+            }
+            defer file.Close()
+
+            c.WriteStream("./" + remoteTarget + "/" + basename, file, 0644)
+
+            counter.Add(1)
+            fmt.Printf(outputContext + ". File %v of %v. Done. (File: %v)\n", counter, total, basename)
+
+            <-semaphore // release
+
+        }()
+
+    }
+
+    wg.Wait()
+    fmt.Println("Done")
+
+}
+
+// Uploads the selected metadata files.
+func UploadMetadataFiles(c *gowebdav.Client, files []string) {
+
+    uploadFiles(c, files, "IMPORT_XML", "Uploading metadata files")
+
+}
+
+// Uploads the selected media files.
+func UploadMediaFiles(c *gowebdav.Client, files []string) {
+
+    uploadFiles(c, files, "IMPORT_IMG", "Uploading media files")
+
+}