Skip to content

Commit 7457183

Browse files
DutchJavaDevDutchJavaDev
authored
Major/fix (#93)
* simba wowwowowhaha * Hot fix ytdlp needing js to solve captch's
1 parent 7197744 commit 7457183

File tree

7 files changed

+282
-77
lines changed

7 files changed

+282
-77
lines changed

‎MyMusicBoxApi/main.go‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package main
22

33
import (
4-
"context"
54
"fmt"
65
"musicboxapi/configuration"
76
"musicboxapi/database"
@@ -12,13 +11,13 @@ import (
1211

1312
"github.com/gin-contrib/cors"
1413
"github.com/gin-gonic/gin"
15-
"github.com/lrstanley/go-ytdlp"
1614
)
1715

1816
// @title MusicBoxApi API
1917
// @version 1.0
2018
// @BasePath /api/v1
2119
func main() {
20+
2221
configuration.LoadConfiguration()
2322

2423
err := database.CreateDatabasConnectionPool()
@@ -33,9 +32,6 @@ func main() {
3332

3433
database.ApplyMigrations()
3534

36-
// If yt-dlp isn't installed yet, download and cache it for further use.
37-
ytdlp.MustInstall(context.TODO(), nil)
38-
3935
setGinMode()
4036

4137
ginEngine := gin.Default()

‎MyMusicBoxApi/release‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ SCRIPTS_FOLDER="$DATABASE_FOLDER/initscripts"
99
IMAGES_FOLDER="$RELEASE_FOLDER/music/images"
1010
COOKIES_FOLDER="$RELEASE_FOLDER/selenium"
1111
MIGRATION_FOLDER="$RELEASE_FOLDER/migration_scripts"
12+
HOTFIX_FOLDER="$RELEASE_FOLDER/hotfix_logs"
1213

1314
echo "=== Setting up release folder ==="
1415
FOLDERS=(
@@ -18,6 +19,7 @@ FOLDERS=(
1819
"$IMAGES_FOLDER"
1920
"$COOKIES_FOLDER"
2021
"$MIGRATION_FOLDER"
22+
"$HOTFIX_FOLDER"
2123
)
2224

2325
for folder in "${FOLDERS[@]}"; do
@@ -37,6 +39,7 @@ cp selenium/* "$COOKIES_FOLDER"
3739
cp migration_scripts/* "$MIGRATION_FOLDER"
3840

3941
echo "=== Building executable ==="
42+
go mod tidy
4043
go build -trimpath -buildvcs=false -ldflags="-s -w" -o "$RELEASE_FOLDER"
4144

4245
echo "=== Reducing executable size ==="

‎MyMusicBoxApi/run‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#!/bin/bash
22
echo "=== Setting up folders ==="
33
IMAGES_FOLDER="music_dev/images"
4+
HOTFIX_FOLDER="music_dev/hotfix_logs"
45

56
FOLDERS=(
67
"$IMAGES_FOLDER"
8+
"$HOTFIX_FOLDER"
79
)
810

911
for folder in "${FOLDERS[@]}"; do

‎MyMusicBoxApi/service/playlist.go‎

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ import (
1111
"strconv"
1212
"strings"
1313
"time"
14-
15-
"github.com/lrstanley/go-ytdlp"
1614
)
1715

1816
func downloadPlaylist(
@@ -26,6 +24,9 @@ func downloadPlaylist(
2624
playlistIdFileName string,
2725
imagesFolder string,
2826
fileExtension string,
27+
logfilesOutputPath string,
28+
logfilesOutputPathError string,
29+
storageFolderName string,
2930
) {
3031
/*
3132
Ignore
@@ -95,21 +96,21 @@ func downloadPlaylist(
9596
_ = os.Rename(oldpath, newpath)
9697
}
9798

98-
defaultSettings := ytdlp.New().
99-
ExtractAudio().
100-
AudioQuality("0").
101-
AudioFormat(fileExtension).
102-
DownloadArchive(archiveFileName).
103-
WriteThumbnail().
104-
ConcurrentFragments(10).
105-
ConvertThumbnails("jpg").
106-
ForceIPv4().
107-
//sudo apt install aria2
108-
Downloader("aria2c").
109-
DownloaderArgs("aria2c:-x 16 -s 16 -j 16").
110-
NoKeepVideo().
111-
Output(storage + "/%(id)s.%(ext)s").
112-
Cookies("selenium/cookies_netscape")
99+
// defaultSettings := ytdlp.New().
100+
// ExtractAudio().
101+
// AudioQuality("0").
102+
// AudioFormat(fileExtension).
103+
// DownloadArchive(archiveFileName).
104+
// WriteThumbnail().
105+
// ConcurrentFragments(10).
106+
// ConvertThumbnails("jpg").
107+
// ForceIPv4().
108+
// //sudo apt install aria2
109+
// Downloader("aria2c").
110+
// DownloaderArgs("aria2c:-x 16 -s 16 -j 16").
111+
// NoKeepVideo().
112+
// Output(storage + "/%(id)s.%(ext)s").
113+
// Cookies("selenium/cookies_netscape")
113114

114115
for id := range downloadCount {
115116
name := names[id]
@@ -121,14 +122,22 @@ func downloadPlaylist(
121122

122123
tasklogTable.UpdateChildTaskLogStatus(childTask)
123124

124-
ytdlpInstance := defaultSettings.Clone()
125-
126-
result, err := ytdlpInstance.Run(context.Background(), fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id]))
127-
128-
// outputLogs[ids[id]] = result.Stdout
129-
130-
if err != nil {
131-
json, _ := json.Marshal(result.Stdout)
125+
downloaded := FlatSingleDownload(
126+
archiveFileName,
127+
idsFileName,
128+
namesFileName,
129+
durationFileName,
130+
playlistTitleFileName,
131+
playlistIdFileName,
132+
fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id]),
133+
logfilesOutputPath,
134+
logfilesOutputPathError,
135+
storageFolderName,
136+
fileExtension)
137+
138+
if !downloaded {
139+
file, err := os.ReadFile(logfilesOutputPathError)
140+
json, _ := json.Marshal(file)
132141
childTask.OutputLog = json
133142
tasklogTable.ChildTaskLogError(childTask)
134143
logging.Error(fmt.Sprintf("Failed to download %s, error:%s", ids[id], err.Error()))
@@ -156,7 +165,13 @@ func downloadPlaylist(
156165

157166
_ = os.Rename(oldpath, newpath)
158167

159-
json, _ := json.Marshal(result.OutputLogs)
168+
file, err := os.ReadFile(logfilesOutputPath)
169+
170+
if err != nil {
171+
panic(-1564654654)
172+
}
173+
174+
json, _ := json.Marshal(file)
160175

161176
childTask.OutputLog = json
162177

‎MyMusicBoxApi/service/ytdlp.go‎

Lines changed: 84 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package service
22

33
import (
44
"bufio"
5-
"context"
65
"encoding/json"
76
"fmt"
87
"io/fs"
@@ -13,8 +12,6 @@ import (
1312
"os"
1413
"strconv"
1514
"strings"
16-
17-
"github.com/lrstanley/go-ytdlp"
1815
)
1916

2017
func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
@@ -34,6 +31,8 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
3431
playlistIdFileName := fmt.Sprintf("%s/playlist_id.%d", storageFolderName, parentTask.Id)
3532
imagesFolder := fmt.Sprintf("%s/images", storageFolderName)
3633
fileExtension := config.OutputExtension
34+
logfilesOutputPath := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrun_%d", parentTask.Id))
35+
logfilesOutputPathError := fmt.Sprintf("%s/hotfix_logs/%s", storageFolderName, fmt.Sprintf("logrunError_%d", parentTask.Id))
3736

3837
if !pathExists(storageFolderName) {
3938
err := os.Mkdir(storageFolderName, fs.ModePerm|fs.ModeDir)
@@ -59,30 +58,31 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
5958
durationFileName,
6059
playlistTitleFileName,
6160
playlistIdFileName,
61+
logfilesOutputPath,
62+
logfilesOutputPathError,
6263
}
6364

6465
if isPlaylist {
65-
dlp := ytdlp.New().
66-
DownloadArchive(archiveFileName).
67-
ForceIPv4().
68-
NoKeepVideo().
69-
SkipDownload().
70-
FlatPlaylist().
71-
WriteThumbnail().
72-
PrintToFile("%(id)s", idsFileName).
73-
PrintToFile("%(title)s", namesFileName).
74-
PrintToFile("%(duration)s", durationFileName).
75-
PrintToFile("%(playlist_title)s", playlistTitleFileName).
76-
PrintToFile("%(playlist_id)s", playlistIdFileName).
77-
Cookies("selenium/cookies_netscape").
78-
IgnoreErrors()
66+
downloaded := FlatPlaylistDownload(
67+
archiveFileName,
68+
idsFileName,
69+
namesFileName,
70+
durationFileName,
71+
playlistTitleFileName,
72+
playlistIdFileName,
73+
downloadRequest.Url,
74+
logfilesOutputPath,
75+
logfilesOutputPathError)
7976

8077
// Start download (flat download)
81-
result, err := dlp.Run(context.Background(), downloadRequest.Url)
78+
if !downloaded {
79+
80+
fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError)
81+
82+
file, err := os.ReadFile(logfilesOutputPathError)
8283

83-
if err != nil {
8484
// Set Task state -> Error
85-
json, err := json.Marshal(result.OutputLogs)
85+
json, err := json.Marshal(string(file))
8686

8787
errChildTask, err := tasklogTable.CreateChildTaskLog(parentTask)
8888

@@ -96,6 +96,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
9696

9797
//Delete created files if any
9898
for _, path := range cleanupFile {
99+
100+
if strings.Contains(path, "logrunError") {
101+
lines, err := readLines(path)
102+
103+
if err != nil {
104+
panic(-654654654)
105+
}
106+
107+
if len(lines) > 0 && len(lines[0]) > 0 {
108+
// skip deleting log so it can be used for debug
109+
continue
110+
}
111+
}
99112
os.Remove(path)
100113
}
101114
return
@@ -133,10 +146,27 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
133146
playlistTitleFileName,
134147
playlistIdFileName,
135148
imagesFolder,
136-
fileExtension)
149+
fileExtension,
150+
logfilesOutputPath,
151+
logfilesOutputPathError,
152+
storageFolderName)
137153

138154
// Delete created files
139155
for _, path := range cleanupFile {
156+
157+
if strings.Contains(path, "logrunError") {
158+
lines, err := readLines(path)
159+
160+
if err != nil {
161+
panic(-654654654)
162+
}
163+
164+
if len(lines) > 0 && len(lines[0]) > 0 {
165+
// skip deleting log so it can be used for debug
166+
continue
167+
}
168+
}
169+
140170
os.Remove(path)
141171
}
142172
} else {
@@ -148,26 +178,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
148178
return
149179
}
150180

151-
dlp := ytdlp.New().
152-
ExtractAudio().
153-
AudioQuality("0").
154-
AudioFormat(fileExtension).
155-
DownloadArchive(archiveFileName).
156-
WriteThumbnail().
157-
ConcurrentFragments(10).
158-
ConvertThumbnails("jpg").
159-
ForceIPv4().
160-
PrintToFile("%(id)s", idsFileName).
161-
PrintToFile("%(title)s", namesFileName).
162-
PrintToFile("%(duration)s", durationFileName).
163-
//sudo apt install aria2
164-
Downloader("aria2c").
165-
DownloaderArgs("aria2c:-x 16 -s 16 -j 16").
166-
NoKeepVideo().
167-
Output(storageFolderName + "/%(id)s.%(ext)s").
168-
Cookies("selenium/cookies_netscape")
181+
downloaded := FlatSingleDownload(
182+
archiveFileName,
183+
idsFileName,
184+
namesFileName,
185+
durationFileName,
186+
playlistTitleFileName,
187+
playlistIdFileName,
188+
downloadRequest.Url,
189+
logfilesOutputPath,
190+
logfilesOutputPathError,
191+
storageFolderName,
192+
fileExtension)
169193

170-
// Update task status
171194
childTask.Status = int(models.Downloading)
172195

173196
err = tasklogTable.UpdateChildTaskLogStatus(childTask)
@@ -176,12 +199,13 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
176199
return
177200
}
178201

179-
// Start download
180-
result, err := dlp.Run(context.Background(), downloadRequest.Url)
202+
if !downloaded {
203+
fmt.Printf("Failed to download for %s check logs at %s", downloadRequest.Url, logfilesOutputPathError)
204+
205+
file, err := os.ReadFile(logfilesOutputPathError)
181206

182-
if err != nil {
183207
// Set Task state -> Error
184-
json, err := json.Marshal(result.OutputLogs)
208+
json, err := json.Marshal(string(file))
185209

186210
childTask.OutputLog = json
187211

@@ -251,7 +275,9 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
251275
logging.Error(fmt.Sprintf("Failed to move song image to / images folder: %s", err.Error()))
252276
}
253277

254-
json, err := json.Marshal(result.OutputLogs)
278+
file, err := os.ReadFile(logfilesOutputPath)
279+
280+
json, err := json.Marshal(string(file))
255281

256282
childTask.OutputLog = json
257283
childTask.Status = int(models.Done)
@@ -263,6 +289,19 @@ func StartDownloadTask(downloadRequest models.DownloadRequestModel) {
263289

264290
//Delete created files
265291
for _, path := range cleanupFile {
292+
293+
if strings.Contains(path, "logrunError") {
294+
lines, err := readLines(path)
295+
296+
if err != nil {
297+
panic(-654654654)
298+
}
299+
300+
if len(lines) > 0 && len(lines[0]) > 0 {
301+
// skip deleting log so it can be used for debug
302+
continue
303+
}
304+
}
266305
os.Remove(path)
267306
}
268307
}

0 commit comments

Comments
 (0)