First working version
- Move from separate dirs for upload to a unified one (identify media/metadata files by file extension) - Prevent uploading when an import is already scheduled - Allow setting custom, parser-specific settings - Add CLI - Implement WebDAV upload - Implement checking of upload folders for uploadable contents Close #6, close #7, close #9, close #3, close #1, close #4
This commit is contained in:
84
src/webdavupload/Upload.go
Normal file
84
src/webdavupload/Upload.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package webdavupload
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"github.com/studio-b12/gowebdav"
|
||||
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
|
||||
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/importconfiggen"
|
||||
)
|
||||
|
||||
// Writes an import config to the remote
|
||||
func SetImportConfigToTheRemote(c *gowebdav.Client, config configloader.MDWebDavUploaderConfig) error {
|
||||
|
||||
importConf := importconfiggen.GenerateImportConfig(config)
|
||||
return c.Write("import_config.txt", []byte(importConf), 0660)
|
||||
|
||||
}
|
||||
|
||||
// Uploads a list of files to the target folder.
|
||||
func uploadFiles(c *gowebdav.Client, files []string, remoteTarget string, outputContext string) {
|
||||
|
||||
total := len(files)
|
||||
var counter atomic.Uint64
|
||||
|
||||
// Determine the number of upload tasks to be processed concurrently.
|
||||
// 10 will be a hard maximum to not spam the server.
|
||||
maxConcTasks := min(10, runtime.NumCPU())
|
||||
|
||||
// Set a semaphore to restrict the number of concurrent upload tasks.
|
||||
semaphore := make(chan struct{}, maxConcTasks)
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
fmt.Printf("Will upload %v files. Processing %v tasks at a time.\n", total, maxConcTasks)
|
||||
|
||||
for _, f := range(files) {
|
||||
|
||||
semaphore <- struct{}{} // acquire
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
|
||||
defer wg.Done()
|
||||
|
||||
basename := filepath.Base(f)
|
||||
|
||||
file, fOpenErr := os.Open(f)
|
||||
if fOpenErr != nil {
|
||||
panic("Failed to read file: " + f)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
c.WriteStream("./" + remoteTarget + "/" + basename, file, 0644)
|
||||
|
||||
counter.Add(1)
|
||||
fmt.Printf(outputContext + ". File %v of %v. Done. (File: %v)\n", counter, total, basename)
|
||||
|
||||
<-semaphore // release
|
||||
|
||||
}()
|
||||
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
fmt.Println("Done")
|
||||
|
||||
}
|
||||
|
||||
// Uploads the selected metadata files.
|
||||
func UploadMetadataFiles(c *gowebdav.Client, files []string) {
|
||||
|
||||
uploadFiles(c, files, "IMPORT_XML", "Uploading metadata files")
|
||||
|
||||
}
|
||||
|
||||
// Uploads the selected media files.
|
||||
func UploadMediaFiles(c *gowebdav.Client, files []string) {
|
||||
|
||||
uploadFiles(c, files, "IMPORT_IMG", "Uploading media files")
|
||||
|
||||
}
|
Reference in New Issue
Block a user