From 4c74234162426855672c13fe540ce9a15db5413d Mon Sep 17 00:00:00 2001 From: DutchJavaDev Date: Thu, 5 Feb 2026 00:11:41 +0000 Subject: [PATCH 1/2] simba wowwowowhaha --- MyMusicBoxApi/main.go | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/MyMusicBoxApi/main.go b/MyMusicBoxApi/main.go index 71ed996..415b545 100644 --- a/MyMusicBoxApi/main.go +++ b/MyMusicBoxApi/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "log" "musicboxapi/configuration" "musicboxapi/database" "musicboxapi/http" @@ -19,6 +20,38 @@ import ( // @version 1.0 // @BasePath /api/v1 func main() { + + procAttr := os.ProcAttr{ + // Dir: "/usr/bin", + } + + procAttr.Files = []*os.File{os.Stdin, os.Stdout, os.Stderr} + + proc, _err := os.StartProcess( + "/usr/bin/yt-dlp-mmb", + []string{ + "yt-dlp-mmb", + "--extractor-args=youtube:player_js_variant=tv", + "--cookies=/home/admin/projects/workspace/MyMusicBoxApi/selenium/cookies_netscape", + "--js-runtimes=deno:/home/admin/.deno/bin", + "--remote-components=ejs:npm", + "https://www.youtube.com/watch?v=Y0ivz1umQ6g", + }, + &os.ProcAttr{ + Files: []*os.File{ + os.Stdin, + os.Stdout, + os.Stderr, + }, + }, + ) + if _err != nil { + log.Fatal(_err) + } + + proc.Wait() + + return configuration.LoadConfiguration() err := database.CreateDatabasConnectionPool() @@ -33,9 +66,16 @@ func main() { database.ApplyMigrations() + ytdlp.RemoveInstallCache() + // If yt-dlp isn't installed yet, download and cache it for further use. - ytdlp.MustInstall(context.TODO(), nil) + options := ytdlp.InstallOptions{ + DisableChecksum: true, + AllowVersionMismatch: true, + DownloadURL: "https://github.com/yt-dlp/yt-dlp/releases/download/2026.01.29/yt-dlp_linux", + } + ytdlp.MustInstall(context.TODO(), &options) setGinMode() ginEngine := gin.Default() From 1be778eeffa0c069dcd5a599bbdf4cf389f8b6ff Mon Sep 17 00:00:00 2001 From: DutchJavaDev Date: Sat, 7 Feb 2026 14:27:12 +0000 Subject: [PATCH 2/2] Hot fix ytdlp needing js to solve captch's --- MyMusicBoxApi/main.go | 44 -------- MyMusicBoxApi/release | 3 + MyMusicBoxApi/run | 2 + MyMusicBoxApi/service/playlist.go | 67 +++++++----- MyMusicBoxApi/service/ytdlp.go | 129 +++++++++++++++-------- MyMusicBoxApi/service/ytdlpHotfix.go | 150 +++++++++++++++++++++++++++ MyMusicBoxApi/stop | 2 +- 7 files changed, 281 insertions(+), 116 deletions(-) create mode 100644 MyMusicBoxApi/service/ytdlpHotfix.go diff --git a/MyMusicBoxApi/main.go b/MyMusicBoxApi/main.go index 415b545..4960f3c 100644 --- a/MyMusicBoxApi/main.go +++ b/MyMusicBoxApi/main.go @@ -1,9 +1,7 @@ package main import ( - "context" "fmt" - "log" "musicboxapi/configuration" "musicboxapi/database" "musicboxapi/http" @@ -13,7 +11,6 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" - "github.com/lrstanley/go-ytdlp" ) // @title MusicBoxApi API @@ -21,37 +18,6 @@ import ( // @BasePath /api/v1 func main() { - procAttr := os.ProcAttr{ - // Dir: "/usr/bin", - } - - procAttr.Files = []*os.File{os.Stdin, os.Stdout, os.Stderr} - - proc, _err := os.StartProcess( - "/usr/bin/yt-dlp-mmb", - []string{ - "yt-dlp-mmb", - "--extractor-args=youtube:player_js_variant=tv", - "--cookies=/home/admin/projects/workspace/MyMusicBoxApi/selenium/cookies_netscape", - "--js-runtimes=deno:/home/admin/.deno/bin", - "--remote-components=ejs:npm", - "https://www.youtube.com/watch?v=Y0ivz1umQ6g", - }, - &os.ProcAttr{ - Files: []*os.File{ - os.Stdin, - os.Stdout, - os.Stderr, - }, - }, - ) - if _err != nil { - log.Fatal(_err) - } - - proc.Wait() - - return configuration.LoadConfiguration() err := database.CreateDatabasConnectionPool() @@ -66,16 +32,6 @@ func main() { database.ApplyMigrations() - ytdlp.RemoveInstallCache() - - // If yt-dlp isn't installed yet, download and cache it for further use. - options := ytdlp.InstallOptions{ - DisableChecksum: true, - AllowVersionMismatch: true, - DownloadURL: "https://github.com/yt-dlp/yt-dlp/releases/download/2026.01.29/yt-dlp_linux", - } - - ytdlp.MustInstall(context.TODO(), &options) setGinMode() ginEngine := gin.Default() diff --git a/MyMusicBoxApi/release b/MyMusicBoxApi/release index 5ee90f6..dc0f269 100755 --- a/MyMusicBoxApi/release +++ b/MyMusicBoxApi/release @@ -9,6 +9,7 @@ SCRIPTS_FOLDER="$DATABASE_FOLDER/initscripts" IMAGES_FOLDER="$RELEASE_FOLDER/music/images" COOKIES_FOLDER="$RELEASE_FOLDER/selenium" MIGRATION_FOLDER="$RELEASE_FOLDER/migration_scripts" +HOTFIX_FOLDER="$RELEASE_FOLDER/hotfix_logs" echo "=== Setting up release folder ===" FOLDERS=( @@ -18,6 +19,7 @@ FOLDERS=( "$IMAGES_FOLDER" "$COOKIES_FOLDER" "$MIGRATION_FOLDER" + "$HOTFIX_FOLDER" ) for folder in "${FOLDERS[@]}"; do @@ -37,6 +39,7 @@ cp selenium/* "$COOKIES_FOLDER" cp migration_scripts/* "$MIGRATION_FOLDER" echo "=== Building executable ===" +go mod tidy go build -trimpath -buildvcs=false -ldflags="-s -w" -o "$RELEASE_FOLDER" echo "=== Reducing executable size ===" diff --git a/MyMusicBoxApi/run b/MyMusicBoxApi/run index c678e44..27b74d2 100755 --- a/MyMusicBoxApi/run +++ b/MyMusicBoxApi/run @@ -1,9 +1,11 @@ #!/bin/bash echo "=== Setting up folders ===" IMAGES_FOLDER="music_dev/images" +HOTFIX_FOLDER="music_dev/hotfix_logs" FOLDERS=( "$IMAGES_FOLDER" + "$HOTFIX_FOLDER" ) for folder in "${FOLDERS[@]}"; do diff --git a/MyMusicBoxApi/service/playlist.go b/MyMusicBoxApi/service/playlist.go index 03c4474..3ae034e 100644 --- a/MyMusicBoxApi/service/playlist.go +++ b/MyMusicBoxApi/service/playlist.go @@ -11,8 +11,6 @@ import ( "strconv" "strings" "time" - - "github.com/lrstanley/go-ytdlp" ) func downloadPlaylist( @@ -26,6 +24,9 @@ func downloadPlaylist( playlistIdFileName string, imagesFolder string, fileExtension string, + logfilesOutputPath string, + logfilesOutputPathError string, + storageFolderName string, ) { /* Ignore @@ -95,21 +96,21 @@ func downloadPlaylist( _ = os.Rename(oldpath, newpath) } - defaultSettings := ytdlp.New(). - ExtractAudio(). - AudioQuality("0"). - AudioFormat(fileExtension). - DownloadArchive(archiveFileName). - WriteThumbnail(). - ConcurrentFragments(10). - ConvertThumbnails("jpg"). - ForceIPv4(). - //sudo apt install aria2 - Downloader("aria2c"). - DownloaderArgs("aria2c:-x 16 -s 16 -j 16"). - NoKeepVideo(). - Output(storage + "/%(id)s.%(ext)s"). - Cookies("selenium/cookies_netscape") + // defaultSettings := ytdlp.New(). + // ExtractAudio(). + // AudioQuality("0"). + // AudioFormat(fileExtension). + // DownloadArchive(archiveFileName). + // WriteThumbnail(). + // ConcurrentFragments(10). + // ConvertThumbnails("jpg"). + // ForceIPv4(). + // //sudo apt install aria2 + // Downloader("aria2c"). + // DownloaderArgs("aria2c:-x 16 -s 16 -j 16"). + // NoKeepVideo(). + // Output(storage + "/%(id)s.%(ext)s"). + // Cookies("selenium/cookies_netscape") for id := range downloadCount { name := names[id] @@ -121,14 +122,22 @@ func downloadPlaylist( tasklogTable.UpdateChildTaskLogStatus(childTask) - ytdlpInstance := defaultSettings.Clone() - - result, err := ytdlpInstance.Run(context.Background(), fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id])) - - // outputLogs[ids[id]] = result.Stdout - - if err != nil { - json, _ := json.Marshal(result.Stdout) + downloaded := FlatSingleDownload( + archiveFileName, + idsFileName, + namesFileName, + durationFileName, + playlistTitleFileName, + playlistIdFileName, + fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id]), + logfilesOutputPath, + logfilesOutputPathError, + storageFolderName, + fileExtension) + + if !downloaded { + file, err := os.ReadFile(logfilesOutputPathError) + json, _ := json.Marshal(file) childTask.OutputLog = json tasklogTable.ChildTaskLogError(childTask) logging.Error(fmt.Sprintf("Failed to download %s, error:%s", ids[id], err.Error())) @@ -156,7 +165,13 @@ func downloadPlaylist( _ = os.Rename(oldpath, newpath) - json, _ := json.Marshal(result.OutputLogs) + file, err := os.ReadFile(logfilesOutputPath) + + if err != nil { + panic(-1564654654) + } + + json, _ := json.Marshal(file) childTask.OutputLog = json diff --git a/MyMusicBoxApi/service/ytdlp.go b/MyMusicBoxApi/service/ytdlp.go index 8fe780b..f4a918c 100644 --- a/MyMusicBoxApi/service/ytdlp.go +++ b/MyMusicBoxApi/service/ytdlp.go @@ -2,7 +2,6 @@ package service import ( "bufio" - "context" "encoding/json" "fmt" "io/fs" @@ -13,8 +12,6 @@ import ( "os" "strconv" "strings" - - "github.com/lrstanley/go-ytdlp" ) func StartDownloadTask(downloadRequest models.DownloadRequestModel) { @@ -34,6 +31,8 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { playlistIdFileName := fmt.Sprintf("%s/playlist_id.%d", storageFolderName, parentTask.Id) imagesFolder := fmt.Sprintf("%s/images", storageFolderName) fileExtension := config.OutputExtension + logfilesOutputPath := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrun_%d", parentTask.Id)) + logfilesOutputPathError := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrunError_%d", parentTask.Id)) if !pathExists(storageFolderName) { err := os.Mkdir(storageFolderName, fs.ModePerm|fs.ModeDir) @@ -59,30 +58,31 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { durationFileName, playlistTitleFileName, playlistIdFileName, + logfilesOutputPath, + logfilesOutputPathError, } if isPlaylist { - dlp := ytdlp.New(). - DownloadArchive(archiveFileName). - ForceIPv4(). - NoKeepVideo(). - SkipDownload(). - FlatPlaylist(). - WriteThumbnail(). - PrintToFile("%(id)s", idsFileName). - PrintToFile("%(title)s", namesFileName). - PrintToFile("%(duration)s", durationFileName). - PrintToFile("%(playlist_title)s", playlistTitleFileName). - PrintToFile("%(playlist_id)s", playlistIdFileName). - Cookies("selenium/cookies_netscape"). - IgnoreErrors() + downloaded := FlatPlaylistDownload( + archiveFileName, + idsFileName, + namesFileName, + durationFileName, + playlistTitleFileName, + playlistIdFileName, + downloadRequest.Url, + logfilesOutputPath, + logfilesOutputPathError) // Start download (flat download) - result, err := dlp.Run(context.Background(), downloadRequest.Url) + if !downloaded { + + fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError) + + file, err := os.ReadFile(logfilesOutputPathError) - if err != nil { // Set Task state -> Error - json, err := json.Marshal(result.OutputLogs) + json, err := json.Marshal(string(file)) errChildTask, err := tasklogTable.CreateChildTaskLog(parentTask) @@ -96,6 +96,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { //Delete created files if any for _, path := range cleanupFile { + + if strings.Contains(path, "logrunError") { + lines, err := readLines(path) + + if err != nil { + panic(-654654654) + } + + if len(lines) > 0 && len(lines[0]) > 0 { + // skip deleting log so it can be used for debug + continue + } + } os.Remove(path) } return @@ -133,10 +146,27 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { playlistTitleFileName, playlistIdFileName, imagesFolder, - fileExtension) + fileExtension, + logfilesOutputPath, + logfilesOutputPathError, + storageFolderName) // Delete created files for _, path := range cleanupFile { + + if strings.Contains(path, "logrunError") { + lines, err := readLines(path) + + if err != nil { + panic(-654654654) + } + + if len(lines) > 0 && len(lines[0]) > 0 { + // skip deleting log so it can be used for debug + continue + } + } + os.Remove(path) } } else { @@ -148,26 +178,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { return } - dlp := ytdlp.New(). - ExtractAudio(). - AudioQuality("0"). - AudioFormat(fileExtension). - DownloadArchive(archiveFileName). - WriteThumbnail(). - ConcurrentFragments(10). - ConvertThumbnails("jpg"). - ForceIPv4(). - PrintToFile("%(id)s", idsFileName). - PrintToFile("%(title)s", namesFileName). - PrintToFile("%(duration)s", durationFileName). - //sudo apt install aria2 - Downloader("aria2c"). - DownloaderArgs("aria2c:-x 16 -s 16 -j 16"). - NoKeepVideo(). - Output(storageFolderName + "/%(id)s.%(ext)s"). - Cookies("selenium/cookies_netscape") + downloaded := FlatSingleDownload( + archiveFileName, + idsFileName, + namesFileName, + durationFileName, + playlistTitleFileName, + playlistIdFileName, + downloadRequest.Url, + logfilesOutputPath, + logfilesOutputPathError, + storageFolderName, + fileExtension) - // Update task status childTask.Status = int(models.Downloading) err = tasklogTable.UpdateChildTaskLogStatus(childTask) @@ -176,12 +199,13 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { return } - // Start download - result, err := dlp.Run(context.Background(), downloadRequest.Url) + if !downloaded { + fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError) + + file, err := os.ReadFile(logfilesOutputPathError) - if err != nil { // Set Task state -> Error - json, err := json.Marshal(result.OutputLogs) + json, err := json.Marshal(string(file)) childTask.OutputLog = json @@ -251,7 +275,9 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { logging.Error(fmt.Sprintf("Failed to move song image to / images folder: %s", err.Error())) } - json, err := json.Marshal(result.OutputLogs) + file, err := os.ReadFile(logfilesOutputPath) + + json, err := json.Marshal(string(file)) childTask.OutputLog = json childTask.Status = int(models.Done) @@ -263,6 +289,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) { //Delete created files for _, path := range cleanupFile { + + if strings.Contains(path, "logrunError") { + lines, err := readLines(path) + + if err != nil { + panic(-654654654) + } + + if len(lines) > 0 && len(lines[0]) > 0 { + // skip deleting log so it can be used for debug + continue + } + } os.Remove(path) } } diff --git a/MyMusicBoxApi/service/ytdlpHotfix.go b/MyMusicBoxApi/service/ytdlpHotfix.go new file mode 100644 index 0000000..dc51a3d --- /dev/null +++ b/MyMusicBoxApi/service/ytdlpHotfix.go @@ -0,0 +1,150 @@ +package service + +import ( + "fmt" + "log" + "os" +) + +var cookiesPath = "selenium/cookies_netscape" + +func FlatPlaylistDownload( + archiveFileName string, + idsFileName string, + namesFileName string, + durationFileName string, + playlistTitleFileName string, + playlistIdFileName string, + url string, + logOutput string, + logOutputError string, +) bool { + + Stdout, err := os.OpenFile(logOutput, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65465465) + } + + Stderr, err := os.OpenFile(logOutputError, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65324465) + } + + proc, _err := os.StartProcess( + "/usr/bin/yt-dlp-mmb", + []string{ + "yt-dlp-mmb", + "--force-ipv4", + "--no-keep-video", + "--skip-download", + "--flat-playlist", + "--write-thumbnail", + "--print-to-file", "%(id)s", idsFileName, + "--print-to-file", "%(title)s", namesFileName, + "--print-to-file", "%(duration)s", durationFileName, + "--print-to-file", "%(playlist_id)s", playlistIdFileName, + "--print-to-file", "%(playlist_title)s", playlistTitleFileName, + "--ignore-errors", + fmt.Sprintf("--download-archive=%s", archiveFileName), + "--extractor-args=youtube:player_js_variant=tv", + fmt.Sprintf("--cookies=%s", cookiesPath), + "--js-runtimes=deno:/home/admin/.deno/bin", + "--remote-components=ejs:npm", + url, + }, + &os.ProcAttr{ + Files: []*os.File{ + os.Stdin, /// :)))))))))))))))))))))))))))))))) + Stdout, + Stderr, + }, + }, + ) + if _err != nil { + log.Fatal(_err) + } + + state, err := proc.Wait() + + if err != nil { + return false + } + + return state.Success() +} + +func FlatSingleDownload( + archiveFileName string, + idsFileName string, + namesFileName string, + durationFileName string, + playlistTitleFileName string, + playlistIdFileName string, + url string, + logOutput string, + logOutputError string, + storageFolderName string, + fileExtension string, +) bool { + + Stdout, err := os.OpenFile(logOutput, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65465465) + } + + Stderr, err := os.OpenFile(logOutputError, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + panic(-65324465) + } + + proc, _err := os.StartProcess( + "/usr/bin/yt-dlp-mmb", + []string{ + "yt-dlp-mmb", + "--force-ipv4", + "--write-thumbnail", + "--extract-audio", + "--audio-quality=0", + fmt.Sprintf("--audio-format=%s", fileExtension), + "--convert-thumbnails=jpg", + "--force-ipv4", + "--downloader=aria2c", + "--no-keep-video", + "--downloader-args=aria2c:-x 16 -s 16 -j 16", + "--print-to-file", "%(id)s", idsFileName, + "--print-to-file", "%(title)s", namesFileName, + "--print-to-file", "%(duration)s", durationFileName, + "--output", storageFolderName + "/%(id)s.%(ext)s", + "--concurrent-fragments=20", + "--ignore-errors", + fmt.Sprintf("--download-archive=%s", archiveFileName), + "--extractor-args=youtube:player_js_variant=tv", + fmt.Sprintf("--cookies=%s", cookiesPath), + "--js-runtimes=deno:/home/admin/.deno/bin", + "--remote-components=ejs:npm", + url, + }, + &os.ProcAttr{ + Files: []*os.File{ + os.Stdin, /// :)))))))))))))))))))))))))))))))) + Stdout, + Stderr, + }, + }, + ) + if _err != nil { + log.Fatal(_err) + } + + state, err := proc.Wait() + + if err != nil { + return false + } + + return state.Success() +} diff --git a/MyMusicBoxApi/stop b/MyMusicBoxApi/stop index d6b7567..dc36eb9 100755 --- a/MyMusicBoxApi/stop +++ b/MyMusicBoxApi/stop @@ -9,7 +9,7 @@ else echo "PID file not found. Process might not be running or already terminated." fi -# kill dev database +## kill dev database cd .. cd dev_database sudo docker compose down \ No newline at end of file