diff --git a/MyMusicBoxApi/main.go b/MyMusicBoxApi/main.go index 71ed996..4960f3c 100644 --- a/MyMusicBoxApi/main.go +++ b/MyMusicBoxApi/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "fmt" "musicboxapi/configuration" "musicboxapi/database" @@ -12,13 +11,13 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" - "github.com/lrstanley/go-ytdlp" ) // @title MusicBoxApi API // @version 1.0 // @BasePath /api/v1 func main() { + configuration.LoadConfiguration() err := database.CreateDatabasConnectionPool() @@ -33,9 +32,6 @@ func main() { database.ApplyMigrations() - // If yt-dlp isn't installed yet, download and cache it for further use. - ytdlp.MustInstall(context.TODO(), nil) - setGinMode() ginEngine := gin.Default() diff --git a/MyMusicBoxApi/release b/MyMusicBoxApi/release index 5ee90f6..dc0f269 100755 --- a/MyMusicBoxApi/release +++ b/MyMusicBoxApi/release @@ -9,6 +9,7 @@ SCRIPTS_FOLDER="$DATABASE_FOLDER/initscripts" IMAGES_FOLDER="$RELEASE_FOLDER/music/images" COOKIES_FOLDER="$RELEASE_FOLDER/selenium" MIGRATION_FOLDER="$RELEASE_FOLDER/migration_scripts" +HOTFIX_FOLDER="$RELEASE_FOLDER/hotfix_logs" echo "=== Setting up release folder ===" FOLDERS=( @@ -18,6 +19,7 @@ FOLDERS=( "$IMAGES_FOLDER" "$COOKIES_FOLDER" "$MIGRATION_FOLDER" + "$HOTFIX_FOLDER" ) for folder in "${FOLDERS[@]}"; do @@ -37,6 +39,7 @@ cp selenium/* "$COOKIES_FOLDER" cp migration_scripts/* "$MIGRATION_FOLDER" echo "=== Building executable ===" +go mod tidy go build -trimpath -buildvcs=false -ldflags="-s -w" -o "$RELEASE_FOLDER" echo "=== Reducing executable size ===" diff --git a/MyMusicBoxApi/run b/MyMusicBoxApi/run index c678e44..27b74d2 100755 --- a/MyMusicBoxApi/run +++ b/MyMusicBoxApi/run @@ -1,9 +1,11 @@ #!/bin/bash echo "=== Setting up folders ===" IMAGES_FOLDER="music_dev/images" +HOTFIX_FOLDER="music_dev/hotfix_logs" FOLDERS=( "$IMAGES_FOLDER" + "$HOTFIX_FOLDER" ) for folder in "${FOLDERS[@]}"; do diff --git a/MyMusicBoxApi/service/playlist.go b/MyMusicBoxApi/service/playlist.go index 03c4474..3ae034e 100644 --- a/MyMusicBoxApi/service/playlist.go +++ b/MyMusicBoxApi/service/playlist.go @@ -11,8 +11,6 @@ import ( "strconv" "strings" "time" - - "github.com/lrstanley/go-ytdlp" ) func downloadPlaylist( @@ -26,6 +24,9 @@ func downloadPlaylist( playlistIdFileName string, imagesFolder string, fileExtension string, + logfilesOutputPath string, + logfilesOutputPathError string, + storageFolderName string, ) { /* Ignore @@ -95,21 +96,21 @@ func downloadPlaylist( _ = os.Rename(oldpath, newpath) } - defaultSettings := ytdlp.New(). - ExtractAudio(). - AudioQuality("0"). - AudioFormat(fileExtension). - DownloadArchive(archiveFileName). - WriteThumbnail(). - ConcurrentFragments(10). - ConvertThumbnails("jpg"). - ForceIPv4(). - //sudo apt install aria2 - Downloader("aria2c"). - DownloaderArgs("aria2c:-x 16 -s 16 -j 16"). - NoKeepVideo(). - Output(storage + "/%(id)s.%(ext)s"). - Cookies("selenium/cookies_netscape") + // defaultSettings := ytdlp.New(). + // ExtractAudio(). + // AudioQuality("0"). + // AudioFormat(fileExtension). + // DownloadArchive(archiveFileName). + // WriteThumbnail(). + // ConcurrentFragments(10). + // ConvertThumbnails("jpg"). + // ForceIPv4(). + // //sudo apt install aria2 + // Downloader("aria2c"). + // DownloaderArgs("aria2c:-x 16 -s 16 -j 16"). + // NoKeepVideo(). + // Output(storage + "/%(id)s.%(ext)s"). + // Cookies("selenium/cookies_netscape") for id := range downloadCount { name := names[id] @@ -121,14 +122,22 @@ func downloadPlaylist( tasklogTable.UpdateChildTaskLogStatus(childTask) - ytdlpInstance := defaultSettings.Clone() - - result, err := ytdlpInstance.Run(context.Background(), fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id])) - - // outputLogs[ids[id]] = result.Stdout - - if err != nil { - json, _ := json.Marshal(result.Stdout) + downloaded := FlatSingleDownload( + archiveFileName, + idsFileName, + namesFileName, + durationFileName, + playlistTitleFileName, + playlistIdFileName, + fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id]), + logfilesOutputPath, + logfilesOutputPathError, + storageFolderName, + fileExtension) + + if !downloaded { + file, err := os.ReadFile(logfilesOutputPathError) + json, _ := json.Marshal(file) childTask.OutputLog = json tasklogTable.ChildTaskLogError(childTask) logging.Error(fmt.Sprintf("Failed to download %s, error:%s", ids[id], err.Error())) @@ -156,7 +165,13 @@ func downloadPlaylist( _ = os.Rename(oldpath, newpath) - json, _ := json.Marshal(result.OutputLogs) + file, err := os.ReadFile(logfilesOutputPath) + + if err != nil { + panic(-1564654654) + } + + json, _ := json.Marshal(file) childTask.OutputLog = json diff --git a/MyMusicBoxApi/service/ytdlp.go b/MyMusicBoxApi/service/ytdlp.go index 8fe780b..f4a918c 100644 --- a/MyMusicBoxApi/service/ytdlp.go +++ b/MyMusicBoxApi/service/ytdlp.go @@ -2,7 +2,6 @@ package service import ( "bufio" - "context" "encoding/json" "fmt" "io/fs" @@ -13,8 +12,6 @@ import ( "os" "strconv" "strings" - - "github.com/lrstanley/go-ytdlp" ) func StartDownloadTask(downloadRequest models.DownloadRequestModel) { @@ -34,6 +31,8 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { playlistIdFileName := fmt.Sprintf("%s/playlist_id.%d", storageFolderName, parentTask.Id) imagesFolder := fmt.Sprintf("%s/images", storageFolderName) fileExtension := config.OutputExtension + logfilesOutputPath := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrun_%d", parentTask.Id)) + logfilesOutputPathError := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrunError_%d", parentTask.Id)) if !pathExists(storageFolderName) { err := os.Mkdir(storageFolderName, fs.ModePerm|fs.ModeDir) @@ -59,30 +58,31 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { durationFileName, playlistTitleFileName, playlistIdFileName, + logfilesOutputPath, + logfilesOutputPathError, } if isPlaylist { - dlp := ytdlp.New(). - DownloadArchive(archiveFileName). - ForceIPv4(). - NoKeepVideo(). - SkipDownload(). - FlatPlaylist(). - WriteThumbnail(). - PrintToFile("%(id)s", idsFileName). - PrintToFile("%(title)s", namesFileName). - PrintToFile("%(duration)s", durationFileName). - PrintToFile("%(playlist_title)s", playlistTitleFileName). - PrintToFile("%(playlist_id)s", playlistIdFileName). - Cookies("selenium/cookies_netscape"). - IgnoreErrors() + downloaded := FlatPlaylistDownload( + archiveFileName, + idsFileName, + namesFileName, + durationFileName, + playlistTitleFileName, + playlistIdFileName, + downloadRequest.Url, + logfilesOutputPath, + logfilesOutputPathError) // Start download (flat download) - result, err := dlp.Run(context.Background(), downloadRequest.Url) + if !downloaded { + + fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError) + + file, err := os.ReadFile(logfilesOutputPathError) - if err != nil { // Set Task state -> Error - json, err := json.Marshal(result.OutputLogs) + json, err := json.Marshal(string(file)) errChildTask, err := tasklogTable.CreateChildTaskLog(parentTask) @@ -96,6 +96,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { //Delete created files if any for _, path := range cleanupFile { + + if strings.Contains(path, "logrunError") { + lines, err := readLines(path) + + if err != nil { + panic(-654654654) + } + + if len(lines) > 0 && len(lines[0]) > 0 { + // skip deleting log so it can be used for debug + continue + } + } os.Remove(path) } return @@ -133,10 +146,27 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { playlistTitleFileName, playlistIdFileName, imagesFolder, - fileExtension) + fileExtension, + logfilesOutputPath, + logfilesOutputPathError, + storageFolderName) // Delete created files for _, path := range cleanupFile { + + if strings.Contains(path, "logrunError") { + lines, err := readLines(path) + + if err != nil { + panic(-654654654) + } + + if len(lines) > 0 && len(lines[0]) > 0 { + // skip deleting log so it can be used for debug + continue + } + } + os.Remove(path) } } else { @@ -148,26 +178,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { return } - dlp := ytdlp.New(). - ExtractAudio(). - AudioQuality("0"). - AudioFormat(fileExtension). - DownloadArchive(archiveFileName). - WriteThumbnail(). - ConcurrentFragments(10). - ConvertThumbnails("jpg"). - ForceIPv4(). - PrintToFile("%(id)s", idsFileName). - PrintToFile("%(title)s", namesFileName). - PrintToFile("%(duration)s", durationFileName). - //sudo apt install aria2 - Downloader("aria2c"). - DownloaderArgs("aria2c:-x 16 -s 16 -j 16"). - NoKeepVideo(). - Output(storageFolderName + "/%(id)s.%(ext)s"). - Cookies("selenium/cookies_netscape") + downloaded := FlatSingleDownload( + archiveFileName, + idsFileName, + namesFileName, + durationFileName, + playlistTitleFileName, + playlistIdFileName, + downloadRequest.Url, + logfilesOutputPath, + logfilesOutputPathError, + storageFolderName, + fileExtension) - // Update task status childTask.Status = int(models.Downloading) err = tasklogTable.UpdateChildTaskLogStatus(childTask) @@ -176,12 +199,13 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { return } - // Start download - result, err := dlp.Run(context.Background(), downloadRequest.Url) + if !downloaded { + fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError) + + file, err := os.ReadFile(logfilesOutputPathError) - if err != nil { // Set Task state -> Error - json, err := json.Marshal(result.OutputLogs) + json, err := json.Marshal(string(file)) childTask.OutputLog = json @@ -251,7 +275,9 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { logging.Error(fmt.Sprintf("Failed to move song image to / images folder: %s", err.Error())) } - json, err := json.Marshal(result.OutputLogs) + file, err := os.ReadFile(logfilesOutputPath) + + json, err := json.Marshal(string(file)) childTask.OutputLog = json childTask.Status = int(models.Done) @@ -263,6 +289,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { //Delete created files for _, path := range cleanupFile { + + if strings.Contains(path, "logrunError") { + lines, err := readLines(path) + + if err != nil { + panic(-654654654) + } + + if len(lines) > 0 && len(lines[0]) > 0 { + // skip deleting log so it can be used for debug + continue + } + } os.Remove(path) } } diff --git a/MyMusicBoxApi/service/ytdlpHotfix.go b/MyMusicBoxApi/service/ytdlpHotfix.go new file mode 100644 index 0000000..dc51a3d --- /dev/null +++ b/MyMusicBoxApi/service/ytdlpHotfix.go @@ -0,0 +1,150 @@ +package service + +import ( + "fmt" + "log" + "os" +) + +var cookiesPath = "selenium/cookies_netscape" + +func FlatPlaylistDownload( + archiveFileName string, + idsFileName string, + namesFileName string, + durationFileName string, + playlistTitleFileName string, + playlistIdFileName string, + url string, + logOutput string, + logOutputError string, +) bool { + + Stdout, err := os.OpenFile(logOutput, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65465465) + } + + Stderr, err := os.OpenFile(logOutputError, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65324465) + } + + proc, _err := os.StartProcess( + "/usr/bin/yt-dlp-mmb", + []string{ + "yt-dlp-mmb", + "--force-ipv4", + "--no-keep-video", + "--skip-download", + "--flat-playlist", + "--write-thumbnail", + "--print-to-file", "%(id)s", idsFileName, + "--print-to-file", "%(title)s", namesFileName, + "--print-to-file", "%(duration)s", durationFileName, + "--print-to-file", "%(playlist_id)s", playlistIdFileName, + "--print-to-file", "%(playlist_title)s", playlistTitleFileName, + "--ignore-errors", + fmt.Sprintf("--download-archive=%s", archiveFileName), + "--extractor-args=youtube:player_js_variant=tv", + fmt.Sprintf("--cookies=%s", cookiesPath), + "--js-runtimes=deno:/home/admin/.deno/bin", + "--remote-components=ejs:npm", + url, + }, + &os.ProcAttr{ + Files: []*os.File{ + os.Stdin, /// :)))))))))))))))))))))))))))))))) + Stdout, + Stderr, + }, + }, + ) + if _err != nil { + log.Fatal(_err) + } + + state, err := proc.Wait() + + if err != nil { + return false + } + + return state.Success() +} + +func FlatSingleDownload( + archiveFileName string, + idsFileName string, + namesFileName string, + durationFileName string, + playlistTitleFileName string, + playlistIdFileName string, + url string, + logOutput string, + logOutputError string, + storageFolderName string, + fileExtension string, +) bool { + + Stdout, err := os.OpenFile(logOutput, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65465465) + } + + Stderr, err := os.OpenFile(logOutputError, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65324465) + } + + proc, _err := os.StartProcess( + "/usr/bin/yt-dlp-mmb", + []string{ + "yt-dlp-mmb", + "--force-ipv4", + "--write-thumbnail", + "--extract-audio", + "--audio-quality=0", + fmt.Sprintf("--audio-format=%s", fileExtension), + "--convert-thumbnails=jpg", + "--force-ipv4", + "--downloader=aria2c", + "--no-keep-video", + "--downloader-args=aria2c:-x 16 -s 16 -j 16", + "--print-to-file", "%(id)s", idsFileName, + "--print-to-file", "%(title)s", namesFileName, + "--print-to-file", "%(duration)s", durationFileName, + "--output", storageFolderName + "/%(id)s.%(ext)s", + "--concurrent-fragments=20", + "--ignore-errors", + fmt.Sprintf("--download-archive=%s", archiveFileName), + "--extractor-args=youtube:player_js_variant=tv", + fmt.Sprintf("--cookies=%s", cookiesPath), + "--js-runtimes=deno:/home/admin/.deno/bin", + "--remote-components=ejs:npm", + url, + }, + &os.ProcAttr{ + Files: []*os.File{ + os.Stdin, /// :)))))))))))))))))))))))))))))))) + Stdout, + Stderr, + }, + }, + ) + if _err != nil { + log.Fatal(_err) + } + + state, err := proc.Wait() + + if err != nil { + return false + } + + return state.Success() +} diff --git a/MyMusicBoxApi/stop b/MyMusicBoxApi/stop index d6b7567..dc36eb9 100755 --- a/MyMusicBoxApi/stop +++ b/MyMusicBoxApi/stop @@ -9,7 +9,7 @@ else echo "PID file not found. Process might not be running or already terminated." fi -# kill dev database +## kill dev database cd .. cd dev_database sudo docker compose down \ No newline at end of file