mirror of
https://github.com/miniflux/v2.git
synced 2025-07-22 17:18:37 +00:00
Move feed parsers packages in reader package
This commit is contained in:
Frédéric Guillot
parent
c26787f476
commit
d5838b6734
14 changed files with 7 additions and 7 deletions
193
reader/atom/atom.go
Normal file
193
reader/atom/atom.go
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package atom
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/miniflux/miniflux2/helper"
|
||||
"github.com/miniflux/miniflux2/model"
|
||||
"github.com/miniflux/miniflux2/reader/date"
|
||||
"github.com/miniflux/miniflux2/reader/processor"
|
||||
"github.com/miniflux/miniflux2/reader/sanitizer"
|
||||
)
|
||||
|
||||
type atomFeed struct {
|
||||
XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"`
|
||||
ID string `xml:"id"`
|
||||
Title string `xml:"title"`
|
||||
Author atomAuthor `xml:"author"`
|
||||
Links []atomLink `xml:"link"`
|
||||
Entries []atomEntry `xml:"entry"`
|
||||
}
|
||||
|
||||
type atomEntry struct {
|
||||
ID string `xml:"id"`
|
||||
Title string `xml:"title"`
|
||||
Updated string `xml:"updated"`
|
||||
Links []atomLink `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content atomContent `xml:"content"`
|
||||
MediaGroup atomMediaGroup `xml:"http://search.yahoo.com/mrss/ group"`
|
||||
Author atomAuthor `xml:"author"`
|
||||
}
|
||||
|
||||
type atomAuthor struct {
|
||||
Name string `xml:"name"`
|
||||
Email string `xml:"email"`
|
||||
}
|
||||
|
||||
type atomLink struct {
|
||||
URL string `xml:"href,attr"`
|
||||
Type string `xml:"type,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
Length string `xml:"length,attr"`
|
||||
}
|
||||
|
||||
type atomContent struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Data string `xml:",chardata"`
|
||||
XML string `xml:",innerxml"`
|
||||
}
|
||||
|
||||
type atomMediaGroup struct {
|
||||
Description string `xml:"http://search.yahoo.com/mrss/ description"`
|
||||
}
|
||||
|
||||
func (a *atomFeed) Transform() *model.Feed {
|
||||
feed := new(model.Feed)
|
||||
feed.FeedURL = getRelationURL(a.Links, "self")
|
||||
feed.SiteURL = getURL(a.Links)
|
||||
feed.Title = sanitizer.StripTags(a.Title)
|
||||
|
||||
if feed.Title == "" {
|
||||
feed.Title = feed.SiteURL
|
||||
}
|
||||
|
||||
for _, entry := range a.Entries {
|
||||
item := entry.Transform()
|
||||
if item.Author == "" {
|
||||
item.Author = getAuthor(a.Author)
|
||||
}
|
||||
|
||||
feed.Entries = append(feed.Entries, item)
|
||||
}
|
||||
|
||||
return feed
|
||||
}
|
||||
|
||||
func (a *atomEntry) Transform() *model.Entry {
|
||||
entry := new(model.Entry)
|
||||
entry.URL = getURL(a.Links)
|
||||
entry.Date = getDate(a)
|
||||
entry.Author = sanitizer.StripTags(getAuthor(a.Author))
|
||||
entry.Hash = getHash(a)
|
||||
entry.Content = processor.ItemContentProcessor(entry.URL, getContent(a))
|
||||
entry.Title = sanitizer.StripTags(strings.Trim(a.Title, " \n\t"))
|
||||
entry.Enclosures = getEnclosures(a)
|
||||
|
||||
if entry.Title == "" {
|
||||
entry.Title = entry.URL
|
||||
}
|
||||
|
||||
return entry
|
||||
}
|
||||
|
||||
func getURL(links []atomLink) string {
|
||||
for _, link := range links {
|
||||
if strings.ToLower(link.Rel) == "alternate" {
|
||||
return link.URL
|
||||
}
|
||||
|
||||
if link.Rel == "" && link.Type == "" {
|
||||
return link.URL
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func getRelationURL(links []atomLink, relation string) string {
|
||||
for _, link := range links {
|
||||
if strings.ToLower(link.Rel) == relation {
|
||||
return link.URL
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func getDate(a *atomEntry) time.Time {
|
||||
if a.Updated != "" {
|
||||
result, err := date.Parse(a.Updated)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func getContent(a *atomEntry) string {
|
||||
if a.Content.Type == "html" || a.Content.Type == "text" {
|
||||
return a.Content.Data
|
||||
}
|
||||
|
||||
if a.Content.Type == "xhtml" {
|
||||
return a.Content.XML
|
||||
}
|
||||
|
||||
if a.Summary != "" {
|
||||
return a.Summary
|
||||
}
|
||||
|
||||
if a.MediaGroup.Description != "" {
|
||||
return a.MediaGroup.Description
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func getHash(a *atomEntry) string {
|
||||
for _, value := range []string{a.ID, getURL(a.Links)} {
|
||||
if value != "" {
|
||||
return helper.Hash(value)
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func getEnclosures(a *atomEntry) model.EnclosureList {
|
||||
enclosures := make(model.EnclosureList, 0)
|
||||
|
||||
for _, link := range a.Links {
|
||||
if strings.ToLower(link.Rel) == "enclosure" {
|
||||
length, _ := strconv.Atoi(link.Length)
|
||||
enclosures = append(enclosures, &model.Enclosure{URL: link.URL, MimeType: link.Type, Size: length})
|
||||
}
|
||||
}
|
||||
|
||||
return enclosures
|
||||
}
|
||||
|
||||
func getAuthor(author atomAuthor) string {
|
||||
if author.Name != "" {
|
||||
return author.Name
|
||||
}
|
||||
|
||||
if author.Email != "" {
|
||||
return author.Email
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
29
reader/atom/parser.go
Normal file
29
reader/atom/parser.go
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package atom
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
|
||||
"github.com/miniflux/miniflux2/errors"
|
||||
"github.com/miniflux/miniflux2/model"
|
||||
|
||||
"golang.org/x/net/html/charset"
|
||||
)
|
||||
|
||||
// Parse returns a normalized feed struct from a Atom feed.
|
||||
func Parse(data io.Reader) (*model.Feed, error) {
|
||||
atomFeed := new(atomFeed)
|
||||
decoder := xml.NewDecoder(data)
|
||||
decoder.CharsetReader = charset.NewReaderLabel
|
||||
|
||||
err := decoder.Decode(atomFeed)
|
||||
if err != nil {
|
||||
return nil, errors.NewLocalizedError("Unable to parse Atom feed: %v.", err)
|
||||
}
|
||||
|
||||
return atomFeed.Transform(), nil
|
||||
}
|
||||
333
reader/atom/parser_test.go
Normal file
333
reader/atom/parser_test.go
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package atom
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/miniflux/miniflux2/errors"
|
||||
)
|
||||
|
||||
func TestParseAtomSample(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
|
||||
<title>Example Feed</title>
|
||||
<link href="http://example.org/"/>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
<author>
|
||||
<name>John Doe</name>
|
||||
</author>
|
||||
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
|
||||
|
||||
<entry>
|
||||
<title>Atom-Powered Robots Run Amok</title>
|
||||
<link href="http://example.org/2003/12/13/atom03"/>
|
||||
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
<summary>Some text.</summary>
|
||||
</entry>
|
||||
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if feed.Title != "Example Feed" {
|
||||
t.Errorf("Incorrect title, got: %s", feed.Title)
|
||||
}
|
||||
|
||||
if feed.FeedURL != "" {
|
||||
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
|
||||
}
|
||||
|
||||
if feed.SiteURL != "http://example.org/" {
|
||||
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
|
||||
}
|
||||
|
||||
if len(feed.Entries) != 1 {
|
||||
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
|
||||
}
|
||||
|
||||
if !feed.Entries[0].Date.Equal(time.Date(2003, time.December, 13, 18, 30, 2, 0, time.UTC)) {
|
||||
t.Errorf("Incorrect entry date, got: %v", feed.Entries[0].Date)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Hash != "3841e5cf232f5111fc5841e9eba5f4b26d95e7d7124902e0f7272729d65601a6" {
|
||||
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
|
||||
}
|
||||
|
||||
if feed.Entries[0].URL != "http://example.org/2003/12/13/atom03" {
|
||||
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Title != "Atom-Powered Robots Run Amok" {
|
||||
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Content != "Some text." {
|
||||
t.Errorf("Incorrect entry content, got: %s", feed.Entries[0].Content)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Author != "John Doe" {
|
||||
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFeedWithoutTitle(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<link rel="alternate" type="text/html" href="https://example.org/"/>
|
||||
<link rel="self" type="application/atom+xml" href="https://example.org/feed"/>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if feed.Title != "https://example.org/" {
|
||||
t.Errorf("Incorrect feed title, got: %s", feed.Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEntryWithoutTitle(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
|
||||
<title>Example Feed</title>
|
||||
<link href="http://example.org/"/>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
<author>
|
||||
<name>John Doe</name>
|
||||
</author>
|
||||
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
|
||||
|
||||
<entry>
|
||||
<link href="http://example.org/2003/12/13/atom03"/>
|
||||
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
<summary>Some text.</summary>
|
||||
</entry>
|
||||
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Title != "http://example.org/2003/12/13/atom03" {
|
||||
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFeedURL(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>Example Feed</title>
|
||||
<link rel="alternate" type="text/html" href="https://example.org/"/>
|
||||
<link rel="self" type="application/atom+xml" href="https://example.org/feed"/>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if feed.SiteURL != "https://example.org/" {
|
||||
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
|
||||
}
|
||||
|
||||
if feed.FeedURL != "https://example.org/feed" {
|
||||
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEntryTitleWithWhitespaces(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>Example Feed</title>
|
||||
<link href="http://example.org/"/>
|
||||
|
||||
<entry>
|
||||
<title>
|
||||
Some Title
|
||||
</title>
|
||||
<link href="http://example.org/2003/12/13/atom03"/>
|
||||
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
<summary>Some text.</summary>
|
||||
</entry>
|
||||
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Title != "Some Title" {
|
||||
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEntryWithAuthorName(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>Example Feed</title>
|
||||
<link href="http://example.org/"/>
|
||||
|
||||
<entry>
|
||||
<link href="http://example.org/2003/12/13/atom03"/>
|
||||
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
<summary>Some text.</summary>
|
||||
<author>
|
||||
<name>Me</name>
|
||||
<email>me@localhost</email>
|
||||
</author>
|
||||
</entry>
|
||||
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Author != "Me" {
|
||||
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEntryWithoutAuthorName(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>Example Feed</title>
|
||||
<link href="http://example.org/"/>
|
||||
|
||||
<entry>
|
||||
<link href="http://example.org/2003/12/13/atom03"/>
|
||||
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
||||
<updated>2003-12-13T18:30:02Z</updated>
|
||||
<summary>Some text.</summary>
|
||||
<author>
|
||||
<name/>
|
||||
<email>me@localhost</email>
|
||||
</author>
|
||||
</entry>
|
||||
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Author != "me@localhost" {
|
||||
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEntryWithEnclosures(t *testing.T) {
|
||||
data := `<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<id>http://www.example.org/myfeed</id>
|
||||
<title>My Podcast Feed</title>
|
||||
<updated>2005-07-15T12:00:00Z</updated>
|
||||
<author>
|
||||
<name>John Doe</name>
|
||||
</author>
|
||||
<link href="http://example.org" />
|
||||
<link rel="self" href="http://example.org/myfeed" />
|
||||
<entry>
|
||||
<id>http://www.example.org/entries/1</id>
|
||||
<title>Atom 1.0</title>
|
||||
<updated>2005-07-15T12:00:00Z</updated>
|
||||
<link href="http://www.example.org/entries/1" />
|
||||
<summary>An overview of Atom 1.0</summary>
|
||||
<link rel="enclosure"
|
||||
type="audio/mpeg"
|
||||
title="MP3"
|
||||
href="http://www.example.org/myaudiofile.mp3"
|
||||
length="1234" />
|
||||
<link rel="enclosure"
|
||||
type="application/x-bittorrent"
|
||||
title="BitTorrent"
|
||||
href="http://www.example.org/myaudiofile.torrent"
|
||||
length="4567" />
|
||||
<content type="xhtml">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml">
|
||||
<h1>Show Notes</h1>
|
||||
<ul>
|
||||
<li>00:01:00 -- Introduction</li>
|
||||
<li>00:15:00 -- Talking about Atom 1.0</li>
|
||||
<li>00:30:00 -- Wrapping up</li>
|
||||
</ul>
|
||||
</div>
|
||||
</content>
|
||||
</entry>
|
||||
</feed>`
|
||||
|
||||
feed, err := Parse(bytes.NewBufferString(data))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if len(feed.Entries) != 1 {
|
||||
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
|
||||
}
|
||||
|
||||
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
|
||||
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
|
||||
}
|
||||
|
||||
if len(feed.Entries[0].Enclosures) != 2 {
|
||||
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
|
||||
}
|
||||
|
||||
if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" {
|
||||
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Enclosures[0].MimeType != "audio/mpeg" {
|
||||
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[0].MimeType)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Enclosures[0].Size != 1234 {
|
||||
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Enclosures[1].URL != "http://www.example.org/myaudiofile.torrent" {
|
||||
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[1].URL)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Enclosures[1].MimeType != "application/x-bittorrent" {
|
||||
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[1].MimeType)
|
||||
}
|
||||
|
||||
if feed.Entries[0].Enclosures[1].Size != 4567 {
|
||||
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[1].Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseInvalidXml(t *testing.T) {
|
||||
data := `garbage`
|
||||
_, err := Parse(bytes.NewBufferString(data))
|
||||
if err == nil {
|
||||
t.Error("Parse should returns an error")
|
||||
}
|
||||
|
||||
if _, ok := err.(errors.LocalizedError); !ok {
|
||||
t.Error("The error returned must be a LocalizedError")
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue