133 lines
3.3 KiB
Go
133 lines
3.3 KiB
Go
package webdavupload
|
|
|
|
import (
|
|
"fmt"
|
|
"path/filepath"
|
|
"os"
|
|
"io"
|
|
"net/http"
|
|
"runtime"
|
|
"sync"
|
|
"sync/atomic"
|
|
"github.com/studio-b12/gowebdav"
|
|
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader"
|
|
"gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/importconfiggen"
|
|
)
|
|
|
|
// Writes an import config to the remote
|
|
func SetImportConfigToTheRemote(c *gowebdav.Client, config configloader.MDWebDavUploaderConfig) error {
|
|
|
|
importConf := importconfiggen.GenerateImportConfig(config)
|
|
return c.Write("import_config.txt", []byte(importConf), 0660)
|
|
|
|
}
|
|
|
|
// Uploads a list of files to the target folder.
|
|
func uploadFiles(c *gowebdav.Client, w io.Writer, files []string, remoteTarget string, outputContext string) {
|
|
|
|
// Check if the io.Writer is a http writer
|
|
_, wImplementsHttpFlusher := interface{}(w).(http.Flusher)
|
|
|
|
total := len(files)
|
|
var counter atomic.Uint64
|
|
|
|
// Determine the number of upload tasks to be processed concurrently.
|
|
// 10 will be a hard maximum to not spam the server.
|
|
maxConcTasks := min(10, runtime.NumCPU())
|
|
|
|
// Set a semaphore to restrict the number of concurrent upload tasks.
|
|
semaphore := make(chan struct{}, maxConcTasks)
|
|
wg := &sync.WaitGroup{}
|
|
|
|
fmt.Fprintf(w, "Will upload %v files. Processing %v tasks at a time.\n", total, maxConcTasks)
|
|
if wImplementsHttpFlusher == true {
|
|
w.(http.Flusher).Flush()
|
|
}
|
|
|
|
for _, f := range(files) {
|
|
|
|
semaphore <- struct{}{} // acquire
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
basename := filepath.Base(f)
|
|
|
|
file, fOpenErr := os.Open(f)
|
|
if fOpenErr != nil {
|
|
panic("Failed to read file: " + f)
|
|
}
|
|
defer file.Close()
|
|
|
|
c.WriteStream("./" + remoteTarget + "/" + basename, file, 0644)
|
|
|
|
counter.Add(1)
|
|
fmt.Fprintf(w, "Uploading %d of %d - File: %s (%s)\n", counter, total, basename, outputContext)
|
|
if wImplementsHttpFlusher == true {
|
|
w.(http.Flusher).Flush()
|
|
}
|
|
|
|
<-semaphore // release
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
fmt.Println("Done")
|
|
|
|
}
|
|
|
|
// Uploads the selected metadata files.
|
|
func UploadMetadataFiles(c *gowebdav.Client, w io.Writer, files []string) {
|
|
|
|
uploadFiles(c, w, files, "IMPORT_XML", "Uploading metadata files")
|
|
|
|
}
|
|
|
|
// Uploads the selected media files.
|
|
func UploadMediaFiles(c *gowebdav.Client, w io.Writer, files []string) {
|
|
|
|
uploadFiles(c, w, files, "IMPORT_IMG", "Uploading media files")
|
|
|
|
}
|
|
|
|
// Removes a list of files.
|
|
func BatchUnlink(w io.Writer, files []string) {
|
|
|
|
_, wImplementsHttpFlusher := interface{}(w).(http.Flusher)
|
|
|
|
maxConcTasks := min(10, runtime.NumCPU())
|
|
|
|
// Set a semaphore to restrict the number of concurrent upload tasks.
|
|
semaphore := make(chan struct{}, maxConcTasks)
|
|
wg := &sync.WaitGroup{}
|
|
|
|
for _, f := range(files) {
|
|
|
|
semaphore <- struct{}{} // acquire
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
err := os.Remove(f)
|
|
if err != nil {
|
|
panic("Failed to delete file " + f)
|
|
}
|
|
fmt.Fprintf(w, "Delete file %s\n", f)
|
|
<-semaphore // release
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
if wImplementsHttpFlusher == true {
|
|
w.(http.Flusher).Flush()
|
|
}
|
|
|
|
}
|