Skip to content

Commit e00dfec

Browse files
AAndyProgramAAndyProgram
committed
2024.1.12.1
API.Instagram: stories (user) downloading with the wrong aspect ratio for some users API.YouTube: fix incorrect opening of a post from the feed; fix wrong date to data parsing; add data downloading by dates DownloadFeedForm: add merging multiple session feeds into one
1 parent 94edf23 commit e00dfec

10 files changed

Lines changed: 167 additions & 42 deletions

File tree

‎Changelog.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,16 @@
1+
# 2024.1.12.1
2+
3+
*2024-01-12*
4+
5+
- Added
6+
- YouTube (SCrawler): data downloading by dates
7+
- Feed: ability to merge multiple session feeds into one
8+
- Feed: remove session number from special feeds
9+
- Fixed
10+
- **Instagram**: stories (user) downloading with the wrong aspect ratio for some users
11+
- YouTube: incorrect opening of a post from the feed
12+
- YouTube: wrong date to data parsing
13+
114
# 2024.1.12.0
215

316
*2024-01-12*

‎SCrawler/API/Instagram/UserData.vb‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -746,10 +746,15 @@ Namespace API.Instagram
746746
Optional ByVal PostOriginUrl As String = Nothing,
747747
Optional ByVal State As UStates = UStates.Unknown, Optional ByVal Attempts As Integer = 0)
748748
Try
749+
Dim maxSize As Func(Of EContainer, Integer) = Function(ByVal _ss As EContainer) As Integer
750+
Dim w% = AConvert(Of Integer)(_ss.Value("width"), 0)
751+
Dim h% = AConvert(Of Integer)(_ss.Value("height"), 0)
752+
Return Math.Max(w, h)
753+
End Function
749754
Dim wrongData As Predicate(Of Sizes) = Function(_ss) _ss.HasError Or _ss.Data.IsEmptyString
750755
Dim img As Predicate(Of EContainer) = Function(_img) Not _img.Name.IsEmptyString AndAlso _img.Name.StartsWith("image_versions") AndAlso _img.Count > 0
751756
Dim vid As Predicate(Of EContainer) = Function(_vid) Not _vid.Name.IsEmptyString AndAlso _vid.Name.StartsWith("video_versions") AndAlso _vid.Count > 0
752-
Dim ss As Func(Of EContainer, Sizes) = Function(_ss) New Sizes(_ss.Value("width"), _ss.Value("url"))
757+
Dim ss As Func(Of EContainer, Sizes) = Function(_ss) New Sizes(maxSize(_ss), _ss.Value("url"))
753758
Dim ssVid As Func(Of EContainer, Sizes) = ss
754759
Dim ssPic As Func(Of EContainer, Sizes) = ss
755760
Dim mDate As Func(Of EContainer, String) = Function(ByVal elem As EContainer) As String

‎SCrawler/API/YouTube/SiteSettings.vb‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,11 @@ Namespace API.YouTube
9090
End Function
9191
Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String
9292
If Not User Is Nothing AndAlso TypeOf User Is UserData Then
93-
Return $"https://{IIf(DirectCast(User, UserData).IsMusic, "music", "www")}.youtube.com/watch?v={Media.Post.ID}"
93+
If DirectCast(User, UserData).IsMusic Or Media.URL_BASE.IsEmptyString Then
94+
Return $"https://{IIf(DirectCast(User, UserData).IsMusic, "music", "www")}.youtube.com/watch?v={Media.Post.ID}"
95+
Else
96+
Return Media.URL_BASE
97+
End If
9498
Else
9599
Return String.Empty
96100
End If

‎SCrawler/API/YouTube/UserData.vb‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -137,29 +137,50 @@ Namespace API.YouTube
137137
Dim list As New List(Of IYouTubeMediaContainer)
138138
Dim url$ = String.Empty
139139
Dim maxDate As Date? = Nothing
140+
Dim __minDate As Date? = DownloadDateFrom
141+
Dim __maxDate As Date? = DownloadDateTo
142+
Dim __getMinDate As Func(Of Date?, Date?) = Function(ByVal dInput As Date?) As Date?
143+
If dInput.HasValue Then
144+
If __minDate.HasValue Then
145+
Return {__minDate.Value, dInput.Value}.Min
146+
Else
147+
Return dInput
148+
End If
149+
ElseIf __minDate.HasValue Then
150+
Return __minDate
151+
Else
152+
Return Nothing
153+
End If
154+
End Function
155+
Dim shortsUrlStandardize As Action(Of IYouTubeMediaContainer, Integer) = Sub(ByVal c As IYouTubeMediaContainer, ByVal ii As Integer)
156+
Dim sUrl$ = $"https://www.youtube.com/shorts/{c.ID}"
157+
'c.URL = sUrl
158+
c.URL_BASE = sUrl
159+
End Sub
140160
Dim nDate As Func(Of Date?, Date?) = Function(ByVal dInput As Date?) As Date?
141161
If dInput.HasValue Then
142162
If dInput.Value.AddDays(3) < Now Then Return dInput.Value.AddDays(1) Else Return dInput
143163
Else
144164
Return Nothing
145165
End If
146166
End Function
147-
Dim fillList As Func(Of Date?, Boolean) = Function(ByVal lDate As Date?) As Boolean
148-
If Not container Is Nothing AndAlso container.HasElements Then
149-
Dim ce As IEnumerable(Of IYouTubeMediaContainer)
150-
ce = container.Elements
151-
If ce.ListExists Then ce = ce.Where(Function(e) e.ObjectType = YouTubeMediaType.Single)
152-
If ce.ListExists AndAlso lDate.HasValue Then _
153-
ce = ce.Where(Function(e) e.DateAdded <= lDate.Value AndAlso
154-
Not e.ID.IsEmptyString AndAlso Not _TempPostsList.Contains(e.ID))
155-
If ce.ListExists Then
156-
maxDate = ce.Max(Function(e) e.DateAdded)
157-
list.AddRange(ce)
158-
Return True
159-
End If
160-
End If
161-
Return False
162-
End Function
167+
Dim fillList As Func(Of Date?, Boolean, Boolean) = Function(ByVal lDate As Date?, ByVal ___isShorts As Boolean) As Boolean
168+
If Not container Is Nothing AndAlso container.HasElements Then
169+
Dim ce As IEnumerable(Of IYouTubeMediaContainer)
170+
ce = container.Elements
171+
If ce.ListExists Then ce = ce.Where(Function(e) e.ObjectType = YouTubeMediaType.Single)
172+
If ce.ListExists AndAlso lDate.HasValue Then _
173+
ce = ce.Where(Function(e) e.DateAdded >= lDate.Value AndAlso
174+
Not e.ID.IsEmptyString AndAlso Not _TempPostsList.Contains(e.ID))
175+
If ce.ListExists Then
176+
maxDate = ce.Max(Function(e) e.DateAdded)
177+
If ___isShorts Then ce.ListForEach(shortsUrlStandardize, EDP.None)
178+
list.AddRange(ce)
179+
Return True
180+
End If
181+
End If
182+
Return False
183+
End Function
163184
Dim applySpecFolder As Action(Of String, Boolean) = Sub(ByVal fName As String, ByVal isPls As Boolean)
164185
If If(container?.Count, 0) > 0 Then _
165186
container.Elements.ForEach(Sub(ByVal el As YouTubeMediaContainerBase)
@@ -175,33 +196,33 @@ Namespace API.YouTube
175196
maxDate = Nothing
176197
LastDownloadDatePlaylist = nDate(LastDownloadDatePlaylist)
177198
url = $"https://{IIf(IsMusic, "music", "www")}.youtube.com/playlist?list={ID}"
178-
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDatePlaylist,, True)
199+
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, __getMinDate(LastDownloadDatePlaylist), __maxDate,, True)
179200
applySpecFolder.Invoke(String.Empty, False)
180-
If fillList.Invoke(LastDownloadDatePlaylist) Then LastDownloadDatePlaylist = If(maxDate, Now)
201+
If fillList.Invoke(LastDownloadDatePlaylist, False) Then LastDownloadDatePlaylist = If(maxDate, Now)
181202
ElseIf YTMediaType = YouTubeMediaType.Channel Then
182203
If IsMusic Or DownloadYTVideos Then
183204
maxDate = Nothing
184205
LastDownloadDateVideos = nDate(LastDownloadDateVideos)
185206
url = $"https://{IIf(IsMusic, "music", "www")}.youtube.com/{IIf(IsMusic Or IsChannelUser, $"{YouTubeFunctions.UserChannelOption}/", "@")}{ID}"
186-
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDateVideos,, True)
207+
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, __getMinDate(LastDownloadDateVideos), __maxDate,, True)
187208
applySpecFolder.Invoke(IIf(IsMusic, String.Empty, "Videos"), False)
188-
If fillList.Invoke(LastDownloadDateVideos) Then LastDownloadDateVideos = If(maxDate, Now)
209+
If fillList.Invoke(LastDownloadDateVideos, False) Then LastDownloadDateVideos = If(maxDate, Now)
189210
End If
190211
If Not IsMusic And DownloadYTShorts Then
191212
maxDate = Nothing
192213
LastDownloadDateShorts = nDate(LastDownloadDateShorts)
193214
url = $"https://www.youtube.com/{IIf(IsChannelUser, $"{YouTubeFunctions.UserChannelOption}/", "@")}{ID}/shorts"
194-
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDateShorts,, True)
215+
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, __getMinDate(LastDownloadDateShorts), __maxDate,, True)
195216
applySpecFolder.Invoke("Shorts", False)
196-
If fillList.Invoke(LastDownloadDateShorts) Then LastDownloadDateShorts = If(maxDate, Now)
217+
If fillList.Invoke(LastDownloadDateShorts, True) Then LastDownloadDateShorts = If(maxDate, Now)
197218
End If
198219
If Not IsMusic And DownloadYTPlaylists Then
199220
maxDate = Nothing
200221
LastDownloadDatePlaylist = nDate(LastDownloadDatePlaylist)
201222
url = $"https://www.youtube.com/{IIf(IsChannelUser, $"{YouTubeFunctions.UserChannelOption}/", "@")}{ID}/playlists"
202-
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDatePlaylist,, True)
223+
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, __getMinDate(LastDownloadDatePlaylist), __maxDate,, True)
203224
applySpecFolder.Invoke("Playlists", True)
204-
If fillList.Invoke(LastDownloadDatePlaylist) Then LastDownloadDatePlaylist = If(maxDate, Now)
225+
If fillList.Invoke(LastDownloadDatePlaylist, False) Then LastDownloadDatePlaylist = If(maxDate, Now)
205226
End If
206227
If Not IsMusic And (DownloadYTCommunityImages Or DownloadYTCommunityVideos) Then DownloadCommunity(String.Empty, Token)
207228
Else

‎SCrawler/Download/Feed/DownloadFeedForm.Designer.vb‎

Lines changed: 17 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎SCrawler/Download/Feed/DownloadFeedForm.resx‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,10 +147,10 @@
147147
<metadata name="MENU_LOAD_SEP_4.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
148148
<value>False</value>
149149
</metadata>
150-
<metadata name="ToolbarTOP.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
151-
<value>17, 17</value>
152-
</metadata>
153150
<metadata name="MENU_LOAD_SEP_5.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
154151
<value>False</value>
155152
</metadata>
153+
<metadata name="ToolbarTOP.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
154+
<value>17, 17</value>
155+
</metadata>
156156
</root>

0 commit comments

Comments
 (0)