package webdavupload import ( "fmt" "path/filepath" "os" "io" "net/http" "runtime" "sync" "sync/atomic" "github.com/studio-b12/gowebdav" "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/configloader" "gitea.armuli.eu/museum-digital/museum-digital-webdav-uploader/src/importconfiggen" ) // Writes an import config to the remote func SetImportConfigToTheRemote(c *gowebdav.Client, config configloader.MDWebDavUploaderConfig) error { importConf := importconfiggen.GenerateImportConfig(config) return c.Write("import_config.txt", []byte(importConf), 0660) } // Uploads a list of files to the target folder. func uploadFiles(c *gowebdav.Client, w io.Writer, files []string, remoteTarget string, outputContext string) { // Check if the io.Writer is a http writer _, wImplementsHttpFlusher := interface{}(w).(http.Flusher) total := len(files) var counter atomic.Uint64 // Determine the number of upload tasks to be processed concurrently. // 10 will be a hard maximum to not spam the server. maxConcTasks := min(10, runtime.NumCPU()) // Set a semaphore to restrict the number of concurrent upload tasks. semaphore := make(chan struct{}, maxConcTasks) wg := &sync.WaitGroup{} fmt.Fprintf(w, "Will upload %v files. Processing %v tasks at a time.\n", total, maxConcTasks) if wImplementsHttpFlusher == true { w.(http.Flusher).Flush() } for _, f := range(files) { semaphore <- struct{}{} // acquire wg.Add(1) go func() { defer wg.Done() basename := filepath.Base(f) file, fOpenErr := os.Open(f) if fOpenErr != nil { panic("Failed to read file: " + f) } defer file.Close() c.WriteStream("./" + remoteTarget + "/" + basename, file, 0644) counter.Add(1) fmt.Fprintf(w, "Uploading %d of %d - File: %s (%s)\n", counter, total, basename, outputContext) if wImplementsHttpFlusher == true { w.(http.Flusher).Flush() } <-semaphore // release }() } wg.Wait() fmt.Println("Done") } // Uploads the selected metadata files. func UploadMetadataFiles(c *gowebdav.Client, w io.Writer, files []string) { uploadFiles(c, w, files, "IMPORT_XML", "Uploading metadata files") } // Uploads the selected media files. func UploadMediaFiles(c *gowebdav.Client, w io.Writer, files []string) { uploadFiles(c, w, files, "IMPORT_IMG", "Uploading media files") } // Removes a list of files. func BatchUnlink(w io.Writer, files []string) { _, wImplementsHttpFlusher := interface{}(w).(http.Flusher) maxConcTasks := min(10, runtime.NumCPU()) // Set a semaphore to restrict the number of concurrent upload tasks. semaphore := make(chan struct{}, maxConcTasks) wg := &sync.WaitGroup{} for _, f := range(files) { semaphore <- struct{}{} // acquire wg.Add(1) go func() { defer wg.Done() err := os.Remove(f) if err != nil { panic("Failed to delete file " + f) } fmt.Fprintf(w, "Delete file %s\n", f) <-semaphore // release }() } wg.Wait() if wImplementsHttpFlusher == true { w.(http.Flusher).Flush() } }