First working version

- Move from separate dirs for upload to a unified one (identify
  media/metadata files by file extension)
- Prevent uploading when an import is already scheduled
- Allow setting custom, parser-specific settings
- Add CLI
- Implement WebDAV upload
- Implement checking of upload folders for uploadable contents

Close #6, close #7, close #9, close #3, close #1, close #4
This commit is contained in:
2025-02-27 17:29:20 +01:00
parent 9a5c432186
commit 7cfd3bb1de
12 changed files with 553 additions and 72 deletions

View File

@@ -0,0 +1,84 @@
package webdavupload
import (
"fmt"
"path/filepath"
"os"
"runtime"
"sync"
"sync/atomic"
"github.com/studio-b12/gowebdav"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/importconfiggen"
)
// Writes an import config to the remote
func SetImportConfigToTheRemote(c *gowebdav.Client, config configloader.MDWebDavUploaderConfig) error {
importConf := importconfiggen.GenerateImportConfig(config)
return c.Write("import_config.txt", []byte(importConf), 0660)
}
// Uploads a list of files to the target folder.
func uploadFiles(c *gowebdav.Client, files []string, remoteTarget string, outputContext string) {
total := len(files)
var counter atomic.Uint64
// Determine the number of upload tasks to be processed concurrently.
// 10 will be a hard maximum to not spam the server.
maxConcTasks := min(10, runtime.NumCPU())
// Set a semaphore to restrict the number of concurrent upload tasks.
semaphore := make(chan struct{}, maxConcTasks)
wg := &sync.WaitGroup{}
fmt.Printf("Will upload %v files. Processing %v tasks at a time.\n", total, maxConcTasks)
for _, f := range(files) {
semaphore <- struct{}{} // acquire
wg.Add(1)
go func() {
defer wg.Done()
basename := filepath.Base(f)
file, fOpenErr := os.Open(f)
if fOpenErr != nil {
panic("Failed to read file: " + f)
}
defer file.Close()
c.WriteStream("./" + remoteTarget + "/" + basename, file, 0644)
counter.Add(1)
fmt.Printf(outputContext + ". File %v of %v. Done. (File: %v)\n", counter, total, basename)
<-semaphore // release
}()
}
wg.Wait()
fmt.Println("Done")
}
// Uploads the selected metadata files.
func UploadMetadataFiles(c *gowebdav.Client, files []string) {
uploadFiles(c, files, "IMPORT_XML", "Uploading metadata files")
}
// Uploads the selected media files.
func UploadMediaFiles(c *gowebdav.Client, files []string) {
uploadFiles(c, files, "IMPORT_IMG", "Uploading media files")
}