Major/fix

MyMusicBoxApi/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"fmt"
 	"musicboxapi/configuration"
 	"musicboxapi/database"
@@ -12,13 +11,13 @@ import (
 
 	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
-	"github.com/lrstanley/go-ytdlp"
 )
 
 // @title MusicBoxApi API
 // @version 1.0
 // @BasePath  /api/v1
 func main() {
+
 	configuration.LoadConfiguration()
 
 	err := database.CreateDatabasConnectionPool()
@@ -33,9 +32,6 @@ func main() {
 
 	database.ApplyMigrations()
 
-	// If yt-dlp isn't installed yet, download and cache it for further use.
-	ytdlp.MustInstall(context.TODO(), nil)
-
 	setGinMode()
 
 	ginEngine := gin.Default()

MyMusicBoxApi/release

@@ -9,6 +9,7 @@ SCRIPTS_FOLDER="$DATABASE_FOLDER/initscripts"
 IMAGES_FOLDER="$RELEASE_FOLDER/music/images"
 COOKIES_FOLDER="$RELEASE_FOLDER/selenium"
 MIGRATION_FOLDER="$RELEASE_FOLDER/migration_scripts"
+HOTFIX_FOLDER="$RELEASE_FOLDER/hotfix_logs"
 
 echo "=== Setting up release folder ==="
 FOLDERS=(
@@ -18,6 +19,7 @@ FOLDERS=(
   "$IMAGES_FOLDER"
   "$COOKIES_FOLDER"
   "$MIGRATION_FOLDER"
+  "$HOTFIX_FOLDER"
 )
 
 for folder in "${FOLDERS[@]}"; do
@@ -37,6 +39,7 @@ cp selenium/* "$COOKIES_FOLDER"
 cp migration_scripts/* "$MIGRATION_FOLDER"
 
 echo "=== Building executable ==="
+go mod tidy
 go build -trimpath -buildvcs=false -ldflags="-s -w" -o "$RELEASE_FOLDER"
 
 echo "=== Reducing executable size ==="

MyMusicBoxApi/run

@@ -1,9 +1,11 @@
 #!/bin/bash
 echo "=== Setting up folders ==="
 IMAGES_FOLDER="music_dev/images"
+HOTFIX_FOLDER="music_dev/hotfix_logs"
 
 FOLDERS=(
   "$IMAGES_FOLDER"
+  "$HOTFIX_FOLDER"
 )
 
 for folder in "${FOLDERS[@]}"; do

MyMusicBoxApi/service/playlist.go

@@ -11,8 +11,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/lrstanley/go-ytdlp"
 )
 
 func downloadPlaylist(
@@ -26,6 +24,9 @@ func downloadPlaylist(
 	playlistIdFileName string,
 	imagesFolder string,
 	fileExtension string,
+	logfilesOutputPath string,
+	logfilesOutputPathError string,
+	storageFolderName string,
 ) {
 	/*
 			Ignore
@@ -95,21 +96,21 @@ func downloadPlaylist(
 		_ = os.Rename(oldpath, newpath)
 	}
 
-	defaultSettings := ytdlp.New().
-		ExtractAudio().
-		AudioQuality("0").
-		AudioFormat(fileExtension).
-		DownloadArchive(archiveFileName).
-		WriteThumbnail().
-		ConcurrentFragments(10).
-		ConvertThumbnails("jpg").
-		ForceIPv4().
-		//sudo apt install aria2
-		Downloader("aria2c").
-		DownloaderArgs("aria2c:-x 16 -s 16 -j 16").
-		NoKeepVideo().
-		Output(storage + "/%(id)s.%(ext)s").
-		Cookies("selenium/cookies_netscape")
+	// defaultSettings := ytdlp.New().
+	// 	ExtractAudio().
+	// 	AudioQuality("0").
+	// 	AudioFormat(fileExtension).
+	// 	DownloadArchive(archiveFileName).
+	// 	WriteThumbnail().
+	// 	ConcurrentFragments(10).
+	// 	ConvertThumbnails("jpg").
+	// 	ForceIPv4().
+	// 	//sudo apt install aria2
+	// 	Downloader("aria2c").
+	// 	DownloaderArgs("aria2c:-x 16 -s 16 -j 16").
+	// 	NoKeepVideo().
+	// 	Output(storage + "/%(id)s.%(ext)s").
+	// 	Cookies("selenium/cookies_netscape")
 
 	for id := range downloadCount {
 		name := names[id]
@@ -121,14 +122,22 @@ func downloadPlaylist(
 
 			tasklogTable.UpdateChildTaskLogStatus(childTask)
 
-			ytdlpInstance := defaultSettings.Clone()
-
-			result, err := ytdlpInstance.Run(context.Background(), fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id]))
-
-			// outputLogs[ids[id]] = result.Stdout
-
-			if err != nil {
-				json, _ := json.Marshal(result.Stdout)
+			downloaded := FlatSingleDownload(
+				archiveFileName,
+				idsFileName,
+				namesFileName,
+				durationFileName,
+				playlistTitleFileName,
+				playlistIdFileName,
+				fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id]),
+				logfilesOutputPath,
+				logfilesOutputPathError,
+				storageFolderName,
+				fileExtension)
+
+			if !downloaded {
+				file, err := os.ReadFile(logfilesOutputPathError)
+				json, _ := json.Marshal(file)
 				childTask.OutputLog = json
 				tasklogTable.ChildTaskLogError(childTask)
 				logging.Error(fmt.Sprintf("Failed to download %s, error:%s", ids[id], err.Error()))
@@ -156,7 +165,13 @@ func downloadPlaylist(
 
 			_ = os.Rename(oldpath, newpath)
 
-			json, _ := json.Marshal(result.OutputLogs)
+			file, err := os.ReadFile(logfilesOutputPath)
+
+			if err != nil {
+				panic(-1564654654)
+			}
+
+			json, _ := json.Marshal(file)
 
 			childTask.OutputLog = json
 

MyMusicBoxApi/service/ytdlp.go

@@ -2,7 +2,6 @@ package service
 
 import (
 	"bufio"
-	"context"
 	"encoding/json"
 	"fmt"
 	"io/fs"
@@ -13,8 +12,6 @@ import (
 	"os"
 	"strconv"
 	"strings"
-
-	"github.com/lrstanley/go-ytdlp"
 )
 
 func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
@@ -34,6 +31,8 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 	playlistIdFileName := fmt.Sprintf("%s/playlist_id.%d", storageFolderName, parentTask.Id)
 	imagesFolder := fmt.Sprintf("%s/images", storageFolderName)
 	fileExtension := config.OutputExtension
+	logfilesOutputPath := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrun_%d", parentTask.Id))
+	logfilesOutputPathError := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrunError_%d", parentTask.Id))
 
 	if !pathExists(storageFolderName) {
 		err := os.Mkdir(storageFolderName, fs.ModePerm|fs.ModeDir)
@@ -59,30 +58,31 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 		durationFileName,
 		playlistTitleFileName,
 		playlistIdFileName,
+		logfilesOutputPath,
+		logfilesOutputPathError,
 	}
 
 	if isPlaylist {
-		dlp := ytdlp.New().
-			DownloadArchive(archiveFileName).
-			ForceIPv4().
-			NoKeepVideo().
-			SkipDownload().
-			FlatPlaylist().
-			WriteThumbnail().
-			PrintToFile("%(id)s", idsFileName).
-			PrintToFile("%(title)s", namesFileName).
-			PrintToFile("%(duration)s", durationFileName).
-			PrintToFile("%(playlist_title)s", playlistTitleFileName).
-			PrintToFile("%(playlist_id)s", playlistIdFileName).
-			Cookies("selenium/cookies_netscape").
-			IgnoreErrors()
+		downloaded := FlatPlaylistDownload(
+			archiveFileName,
+			idsFileName,
+			namesFileName,
+			durationFileName,
+			playlistTitleFileName,
+			playlistIdFileName,
+			downloadRequest.Url,
+			logfilesOutputPath,
+			logfilesOutputPathError)
 
 		// Start download (flat download)
-		result, err := dlp.Run(context.Background(), downloadRequest.Url)
+		if !downloaded {
+
+			fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError)
+
+			file, err := os.ReadFile(logfilesOutputPathError)
 
-		if err != nil {
 			// Set Task state -> Error
-			json, err := json.Marshal(result.OutputLogs)
+			json, err := json.Marshal(string(file))
 
 			errChildTask, err := tasklogTable.CreateChildTaskLog(parentTask)
 
@@ -96,6 +96,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 
 			//Delete created files if any
 			for _, path := range cleanupFile {
+
+				if strings.Contains(path, "logrunError") {
+					lines, err := readLines(path)
+
+					if err != nil {
+						panic(-654654654)
+					}
+
+					if len(lines) > 0 && len(lines[0]) > 0 {
+						// skip deleting log so it can be used for debug
+						continue
+					}
+				}
 				os.Remove(path)
 			}
 			return
@@ -133,10 +146,27 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 			playlistTitleFileName,
 			playlistIdFileName,
 			imagesFolder,
-			fileExtension)
+			fileExtension,
+			logfilesOutputPath,
+			logfilesOutputPathError,
+			storageFolderName)
 
 		// Delete created files
 		for _, path := range cleanupFile {
+
+			if strings.Contains(path, "logrunError") {
+				lines, err := readLines(path)
+
+				if err != nil {
+					panic(-654654654)
+				}
+
+				if len(lines) > 0 && len(lines[0]) > 0 {
+					// skip deleting log so it can be used for debug
+					continue
+				}
+			}
+
 			os.Remove(path)
 		}
 	} else {
@@ -148,26 +178,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 			return
 		}
 
-		dlp := ytdlp.New().
-			ExtractAudio().
-			AudioQuality("0").
-			AudioFormat(fileExtension).
-			DownloadArchive(archiveFileName).
-			WriteThumbnail().
-			ConcurrentFragments(10).
-			ConvertThumbnails("jpg").
-			ForceIPv4().
-			PrintToFile("%(id)s", idsFileName).
-			PrintToFile("%(title)s", namesFileName).
-			PrintToFile("%(duration)s", durationFileName).
-			//sudo apt install aria2
-			Downloader("aria2c").
-			DownloaderArgs("aria2c:-x 16 -s 16 -j 16").
-			NoKeepVideo().
-			Output(storageFolderName + "/%(id)s.%(ext)s").
-			Cookies("selenium/cookies_netscape")
+		downloaded := FlatSingleDownload(
+			archiveFileName,
+			idsFileName,
+			namesFileName,
+			durationFileName,
+			playlistTitleFileName,
+			playlistIdFileName,
+			downloadRequest.Url,
+			logfilesOutputPath,
+			logfilesOutputPathError,
+			storageFolderName,
+			fileExtension)
 
-		// Update task status
 		childTask.Status = int(models.Downloading)
 
 		err = tasklogTable.UpdateChildTaskLogStatus(childTask)
@@ -176,12 +199,13 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 			return
 		}
 
-		// Start download
-		result, err := dlp.Run(context.Background(), downloadRequest.Url)
+		if !downloaded {
+			fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError)
+
+			file, err := os.ReadFile(logfilesOutputPathError)
 
-		if err != nil {
 			// Set Task state -> Error
-			json, err := json.Marshal(result.OutputLogs)
+			json, err := json.Marshal(string(file))
 
 			childTask.OutputLog = json
 
@@ -251,7 +275,9 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 			logging.Error(fmt.Sprintf("Failed to move song image to / images folder: %s", err.Error()))
 		}
 
-		json, err := json.Marshal(result.OutputLogs)
+		file, err := os.ReadFile(logfilesOutputPath)
+
+		json, err := json.Marshal(string(file))
 
 		childTask.OutputLog = json
 		childTask.Status = int(models.Done)
@@ -263,6 +289,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
 
 		//Delete created files
 		for _, path := range cleanupFile {
+
+			if strings.Contains(path, "logrunError") {
+				lines, err := readLines(path)
+
+				if err != nil {
+					panic(-654654654)
+				}
+
+				if len(lines) > 0 && len(lines[0]) > 0 {
+					// skip deleting log so it can be used for debug
+					continue
+				}
+			}
 			os.Remove(path)
 		}
 	}