oss/miniflux-v2
oss/miniflux-v2
1
0
Fork 0
mirror of https://github.com/miniflux/v2.git synced 2025-09-15 18:57:04 +00:00
Code Wiki Activity

Improve YouTube page feed detection

Browse source 
In order to be more resilient to YouTube URLs variation and
to address this feature_request: https://github.com/miniflux/v2/issues/2628
I've reworked a bit the way the YouTube feed extraction is done.

I've kept all the `FindSubscriptionsFromYouTube*` in order
to keep all the existing unit tests as-is ensuring little to no
regressions. By doing so, I had to call twice `youtubeURLIDExtractor`.
Small performance penalty for peace of mind in my opinion.

`youtubeURLIDExtractor` is made in a way only one kind
of page can be detected at a time. This mean I can
solve the "video in a playlist" feature_request
by prioritizing the playlist ID over the Video ID

Also, by using `url.Parse()` to get ids, it's safer
to url mangle and variation. The most common variation
being the `t=42` parameters that start the playback
at a given position. Previously, this kind of url
would not be detected as "YouTube URL".

I deliberately ignored the url parsing error
to keep previous behavior (skip the YouTube analysis and follow with the other analysis)

I also tried to keep debug logs the same as before as much as I could.

I manually tested all the YouTube cases (video,channel,playlist)
and they all work as expected except for the video. But this one
does not work either on main. The `meta` html tag that was searched for
does not seem to exist anymore.

fix: #2628
This commit is contained in:
Ztec 2024-06-13 16:30:15 +02:00 • committed by Frédéric Guillot
parent 07f6d397d4
commit e54825bf02
2 changed files with 165 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

71
internal/reader/subscription/finder_test.go
View file

@ -4,14 +4,16 @@
package subscription
import (
"errors"
"strings"
"testing"
)
func TestFindYoutubePlaylistFeed(t *testing.T) {
scenarios := map[string]string{
"https://www.youtube.com/playlist?list=PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR",
"https://www.youtube.com/playlist?list=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM",
"https://www.youtube.com/playlist?list=PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR",
"https://www.youtube.com/playlist?list=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM",
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM": "https://www.youtube.com/feeds/videos.xml?playlist_id=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM",
}
for websiteURL, expectedFeedURL := range scenarios {
@ -30,6 +32,71 @@ func TestFindYoutubePlaylistFeed(t *testing.T) {
}
}
func TestItDoesNotConsiderPlaylistWatchPageAsVideoWatchPage(t *testing.T) {
_, localizedError := NewSubscriptionFinder(nil).FindSubscriptionsFromYouTubeVideoPage("https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM")
if localizedError != nil {
t.Fatalf(`Should not consider a playlist watch page as a video watch page`)
}
}
func TestYoutubeIdExtractor(t *testing.T) {
type testResult struct {
ID string
Kind youtubeKind
error error
}
urls := map[string]testResult{
"https://www.youtube.com/watch?v=dQw4w9WgXcQ": {
ID: "dQw4w9WgXcQ",
Kind: youtubeIDKindVideo,
error: nil,
},
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=1": {
ID: "dQw4w9WgXcQ",
Kind: youtubeIDKindVideo,
error: nil,
},
"https://www.youtube.com/watch?t=1&v=dQw4w9WgXcQ": {
ID: "dQw4w9WgXcQ",
Kind: youtubeIDKindVideo,
error: nil,
},
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM": {
ID: "PLOOwEPgFWm_N42HlCLhqyJ0ZBWr5K1QDM",
Kind: youtubeIDKindPlaylist,
error: nil,
},
"https://www.youtube.com/playlist?list=PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR": {
ID: "PLOOwEPgFWm_NHcQd9aCi5JXWASHO_n5uR",
Kind: youtubeIDKindPlaylist,
error: nil,
},
"https://www.youtube.com/channel/UC-Qj80avWItNRjkZ41rzHyw": {
ID: "UC-Qj80avWItNRjkZ41rzHyw",
Kind: youtubeIDKindChannel,
error: nil,
},
"https://www.example.com/channel/UC-Qj80avWItNRjkZ41rzHyw": {
ID: "",
Kind: "",
error: errNotYoutubeUrl,
},
}
for websiteURL, expected := range urls {
kind, id, err := youtubeURLIDExtractor(websiteURL)
if !errors.Is(err, expected.error) {
t.Fatalf(`Unexpected error: %v got %v`, expected.error, err)
}
if id != expected.ID {
t.Fatalf(`Unexpected ID: %v got %v`, expected.ID, id)
}
if kind != expected.Kind {
t.Fatalf(`Unexpected Kind: %v got %v`, expected.Kind, kind)
}
}
}
func TestFindYoutubeChannelFeed(t *testing.T) {
scenarios := map[string]string{
"https://www.youtube.com/channel/UC-Qj80avWItNRjkZ41rzHyw": "https://www.youtube.com/feeds/videos.xml?channel_id=UC-Qj80avWItNRjkZ41rzHyw",