Add the possibility to override default user agent for each feed

api/feed.go

@@ -47,6 +47,7 @@ func (c *Controller) CreateFeed(w http.ResponseWriter, r *http.Request) {
 		feedInfo.CategoryID,
 		feedInfo.FeedURL,
 		feedInfo.Crawler,
+		feedInfo.UserAgent,
 		feedInfo.Username,
 		feedInfo.Password,
 	)

api/payload.go

@@ -26,15 +26,17 @@ type entriesResponse struct {
 type feedCreation struct {
 	FeedURL    string `json:"feed_url"`
 	CategoryID int64  `json:"category_id"`
+	UserAgent  string `json:"user_agent"`
 	Username   string `json:"username"`
 	Password   string `json:"password"`
 	Crawler    bool   `json:"crawler"`
 }
 
 type subscriptionDiscovery struct {
-	URL      string `json:"url"`
+	URL       string `json:"url"`
-	Username string `json:"username"`
+	UserAgent string `json:"user_agent"`
-	Password string `json:"password"`
+	Username  string `json:"username"`
+	Password  string `json:"password"`
 }
 
 type feedModification struct {
@@ -44,6 +46,7 @@ type feedModification struct {
 	ScraperRules *string `json:"scraper_rules"`
 	RewriteRules *string `json:"rewrite_rules"`
 	Crawler      *bool   `json:"crawler"`
+	UserAgent    *string `json:"user_agent"`
 	Username     *string `json:"username"`
 	Password     *string `json:"password"`
 	CategoryID   *int64  `json:"category_id"`
@@ -74,6 +77,10 @@ func (f *feedModification) Update(feed *model.Feed) {
 		feed.Crawler = *f.Crawler
 	}
 
+	if f.UserAgent != nil {
+		feed.UserAgent = *f.UserAgent
+	}
+
 	if f.Username != nil {
 		feed.Username = *f.Username
 	}

api/subscription.go

@@ -22,6 +22,7 @@ func (c *Controller) GetSubscriptions(w http.ResponseWriter, r *http.Request) {
 
 	subscriptions, err := subscription.FindSubscriptions(
 		subscriptionInfo.URL,
+		subscriptionInfo.UserAgent,
 		subscriptionInfo.Username,
 		subscriptionInfo.Password,
 	)

client/core.go

@@ -91,6 +91,7 @@ type Feed struct {
 	ScraperRules       string    `json:"scraper_rules"`
 	RewriteRules       string    `json:"rewrite_rules"`
 	Crawler            bool      `json:"crawler"`
+	UserAgent          string    `json:"user_agent"`
 	Username           string    `json:"username"`
 	Password           string    `json:"password"`
 	Category           *Category `json:"category,omitempty"`
@@ -105,6 +106,7 @@ type FeedModification struct {
 	ScraperRules *string `json:"scraper_rules"`
 	RewriteRules *string `json:"rewrite_rules"`
 	Crawler      *bool   `json:"crawler"`
+	UserAgent    *string `json:"user_agent"`
 	Username     *string `json:"username"`
 	Password     *string `json:"password"`
 	CategoryID   *int64  `json:"category_id"`

database/migration.go

@@ -12,7 +12,7 @@ import (
 	"miniflux.app/logger"
 )
 
-const schemaVersion = 20
+const schemaVersion = 21
 
 // Migrate executes database migrations.
 func Migrate(db *sql.DB) {

database/sql.go

@@ -144,6 +144,7 @@ create index users_extra_idx on users using gin(extra);
 	"schema_version_20": `alter table entries add column document_vectors tsvector;
 update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, ''));
 create index document_vectors_idx on entries using gin(document_vectors);`,
+	"schema_version_21": `alter table feeds add column user_agent text default '';`,
 	"schema_version_3": `create table tokens (
     id text not null,
     value text not null,
@@ -192,6 +193,7 @@ var SqlMapChecksums = map[string]string{
 	"schema_version_19": "a83f77b41cc213d282805a5b518f15abbf96331599119f0ef4aca4be037add7b",
 	"schema_version_2":  "e8e9ff32478df04fcddad10a34cba2e8bb1e67e7977b5bd6cdc4c31ec94282b4",
 	"schema_version_20": "6c4e9b2c5bccdc3243c239c390fb1caa5e15624e669b2c07e14c126f6d2e2cd6",
+	"schema_version_21": "77da01ee38918ff4fe33985fbb20ed3276a717a7584c2ca9ebcf4d4ab6cb6910",
 	"schema_version_3":  "a54745dbc1c51c000f74d4e5068f1e2f43e83309f023415b1749a47d5c1e0f12",
 	"schema_version_4":  "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
 	"schema_version_5":  "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",

database/sql/schema_version_21.sql

@@ -0,0 +1 @@
+alter table feeds add column user_agent text default '';

http/client/client.go

@@ -33,6 +33,9 @@ const (
 )
 
 var (
+	// DefaultUserAgent sets the User-Agent header used for any requests by miniflux.
+	DefaultUserAgent = "Mozilla/5.0 (compatible; Miniflux/" + version.Version + "; +https://miniflux.app)"
+
 	errInvalidCertificate        = "Invalid SSL certificate (original error: %q)"
 	errTemporaryNetworkOperation = "This website is temporarily unreachable (original error: %q)"
 	errPermanentNetworkOperation = "This website is permanently unreachable (original error: %q)"
@@ -47,6 +50,7 @@ type Client struct {
 	authorizationHeader string
 	username            string
 	password            string
+	userAgent           string
 	Insecure            bool
 }
 
@@ -72,6 +76,14 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client
 	return c
 }
 
+// WithUserAgent defines the User-Agent header to use for outgoing requests.
+func (c *Client) WithUserAgent(userAgent string) *Client {
+	if userAgent != "" {
+		c.userAgent = userAgent
+	}
+	return c
+}
+
 // Get execute a GET HTTP request.
 func (c *Client) Get() (*Response, error) {
 	request, err := c.buildRequest(http.MethodGet, nil)
@@ -212,7 +224,7 @@ func (c *Client) buildClient() http.Client {
 
 func (c *Client) buildHeaders() http.Header {
 	headers := make(http.Header)
-	headers.Add("User-Agent", "Mozilla/5.0 (compatible; Miniflux/"+version.Version+"; +https://miniflux.app)")
+	headers.Add("User-Agent", c.userAgent)
 	headers.Add("Accept", "*/*")
 
 	if c.etagHeader != "" {
@@ -233,5 +245,5 @@ func (c *Client) buildHeaders() http.Header {
 
 // New returns a new HTTP client.
 func New(url string) *Client {
-	return &Client{url: url, Insecure: false}
+	return &Client{url: url, userAgent: DefaultUserAgent, Insecure: false}
 }

model/feed.go

@@ -24,6 +24,7 @@ type Feed struct {
 	ScraperRules       string    `json:"scraper_rules"`
 	RewriteRules       string    `json:"rewrite_rules"`
 	Crawler            bool      `json:"crawler"`
+	UserAgent          string    `json:"user_agent"`
 	Username           string    `json:"username"`
 	Password           string    `json:"password"`
 	Category           *Category `json:"category,omitempty"`

reader/feed/handler.go

@@ -37,7 +37,7 @@ type Handler struct {
 }
 
 // CreateFeed fetch, parse and store a new feed.
-func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, username, password string) (*model.Feed, error) {
+func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password string) (*model.Feed, error) {
 	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
 
 	if !h.store.CategoryExists(userID, categoryID) {
@@ -46,6 +46,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
 
 	clt := client.New(url)
 	clt.WithCredentials(username, password)
+	clt.WithUserAgent(userAgent)
 	response, err := clt.Get()
 	if err != nil {
 		if _, ok := err.(*errors.LocalizedError); ok {
@@ -87,6 +88,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
 	subscription.FeedURL = response.EffectiveURL
 	subscription.UserID = userID
 	subscription.Crawler = crawler
+	subscription.UserAgent = userAgent
 	subscription.Username = username
 	subscription.Password = password
 
@@ -136,6 +138,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
 	clt := client.New(originalFeed.FeedURL)
 	clt.WithCredentials(originalFeed.Username, originalFeed.Password)
 	clt.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
+	clt.WithUserAgent(originalFeed.UserAgent)
 	response, err := clt.Get()
 	if err != nil {
 		var customErr errors.LocalizedError
@@ -196,6 +199,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
 
 		feedProcessor := processor.NewFeedProcessor(userID, h.store, subscription)
 		feedProcessor.WithScraperRules(originalFeed.ScraperRules)
+		feedProcessor.WithUserAgent(originalFeed.UserAgent)
 		feedProcessor.WithRewriteRules(originalFeed.RewriteRules)
 		feedProcessor.WithCrawler(originalFeed.Crawler)
 		feedProcessor.Process()

reader/processor/processor.go

@@ -21,6 +21,7 @@ type FeedProcessor struct {
 	scraperRules string
 	rewriteRules string
 	crawler      bool
+	userAgent    string
 }
 
 // WithCrawler enables the crawler.
@@ -33,6 +34,11 @@ func (f *FeedProcessor) WithScraperRules(rules string) {
 	f.scraperRules = rules
 }
 
+// WithUserAgent sets the User-Agent header for fetching article content.
+func (f *FeedProcessor) WithUserAgent(userAgent string) {
+	f.userAgent = userAgent
+}
+
 // WithRewriteRules adds rewrite rules to the processing.
 func (f *FeedProcessor) WithRewriteRules(rules string) {
 	f.rewriteRules = rules
@@ -45,7 +51,7 @@ func (f *FeedProcessor) Process() {
 			if f.store.EntryURLExists(f.userID, entry.URL) {
 				logger.Debug(`[FeedProcessor] Do not crawl existing entry URL: "%s"`, entry.URL)
 			} else {
-				content, err := scraper.Fetch(entry.URL, f.scraperRules)
+				content, err := scraper.Fetch(entry.URL, f.scraperRules, f.userAgent)
 				if err != nil {
 					logger.Error("[FeedProcessor] %v", err)
 				} else {

reader/scraper/scraper.go

@@ -19,8 +19,12 @@ import (
 )
 
 // Fetch downloads a web page a returns relevant contents.
-func Fetch(websiteURL, rules string) (string, error) {
+func Fetch(websiteURL, rules, userAgent string) (string, error) {
 	clt := client.New(websiteURL)
+	if userAgent != "" {
+		clt.WithUserAgent(userAgent)
+	}
+
 	response, err := clt.Get()
 	if err != nil {
 		return "", err

reader/subscription/finder.go

@@ -29,11 +29,12 @@ var (
 )
 
 // FindSubscriptions downloads and try to find one or more subscriptions from an URL.
-func FindSubscriptions(websiteURL, username, password string) (Subscriptions, error) {
+func FindSubscriptions(websiteURL, userAgent, username, password string) (Subscriptions, error) {
 	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[FindSubscriptions] url=%s", websiteURL))
 
 	clt := client.New(websiteURL)
 	clt.WithCredentials(username, password)
+	clt.WithUserAgent(userAgent)
 	response, err := clt.Get()
 	if err != nil {
 		if _, ok := err.(errors.LocalizedError); ok {

storage/entry_query_builder.go

@@ -192,7 +192,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
 		e.id, e.user_id, e.feed_id, e.hash, e.published_at at time zone u.timezone, e.title,
 		e.url, e.comments_url, e.author, e.content, e.status, e.starred,
 		f.title as feed_title, f.feed_url, f.site_url, f.checked_at,
-		f.category_id, c.title as category_title, f.scraper_rules, f.rewrite_rules, f.crawler,
+		f.category_id, c.title as category_title, f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent,
 		fi.icon_id,
 		u.timezone
 		FROM entries e
@@ -247,6 +247,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
 			&entry.Feed.ScraperRules,
 			&entry.Feed.RewriteRules,
 			&entry.Feed.Crawler,
+			&entry.Feed.UserAgent,
 			&iconID,
 			&tz,
 		)

storage/feed.go

@@ -66,7 +66,7 @@ func (s *Storage) Feeds(userID int64) (model.Feeds, error) {
 		f.id, f.feed_url, f.site_url, f.title, f.etag_header, f.last_modified_header,
 		f.user_id, f.checked_at at time zone u.timezone,
 		f.parsing_error_count, f.parsing_error_msg,
-		f.scraper_rules, f.rewrite_rules, f.crawler,
+		f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent,
 		f.username, f.password,
 		f.category_id, c.title as category_title,
 		fi.icon_id,
@@ -104,6 +104,7 @@ func (s *Storage) Feeds(userID int64) (model.Feeds, error) {
 			&feed.ScraperRules,
 			&feed.RewriteRules,
 			&feed.Crawler,
+			&feed.UserAgent,
 			&feed.Username,
 			&feed.Password,
 			&feed.Category.ID,
@@ -141,7 +142,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
 		f.id, f.feed_url, f.site_url, f.title, f.etag_header, f.last_modified_header,
 		f.user_id, f.checked_at at time zone u.timezone,
 		f.parsing_error_count, f.parsing_error_msg,
-		f.scraper_rules, f.rewrite_rules, f.crawler,
+		f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent,
 		f.username, f.password,
 		f.category_id, c.title as category_title,
 		fi.icon_id,
@@ -166,6 +167,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
 		&feed.ScraperRules,
 		&feed.RewriteRules,
 		&feed.Crawler,
+		&feed.UserAgent,
 		&feed.Username,
 		&feed.Password,
 		&feed.Category.ID,
@@ -194,8 +196,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:CreateFeed] feedURL=%s", feed.FeedURL))
 	sql := `
 		INSERT INTO feeds
-		(feed_url, site_url, title, category_id, user_id, etag_header, last_modified_header, crawler, username, password)
+		(feed_url, site_url, title, category_id, user_id, etag_header, last_modified_header, crawler, user_agent, username, password)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
 		RETURNING id
 	`
 
@@ -209,6 +211,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 		feed.EtagHeader,
 		feed.LastModifiedHeader,
 		feed.Crawler,
+		feed.UserAgent,
 		feed.Username,
 		feed.Password,
 	).Scan(&feed.ID)
@@ -234,9 +237,9 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
 
 	query := `UPDATE feeds SET
 		feed_url=$1, site_url=$2, title=$3, category_id=$4, etag_header=$5, last_modified_header=$6, checked_at=$7,
-		parsing_error_msg=$8, parsing_error_count=$9, scraper_rules=$10, rewrite_rules=$11, crawler=$12,
+		parsing_error_msg=$8, parsing_error_count=$9, scraper_rules=$10, rewrite_rules=$11, crawler=$12, user_agent=$13,
-		username=$13, password=$14
+		username=$14, password=$15
-		WHERE id=$15 AND user_id=$16`
+		WHERE id=$16 AND user_id=$17`
 
 	_, err = s.db.Exec(query,
 		feed.FeedURL,
@@ -251,6 +254,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
 		feed.ScraperRules,
 		feed.RewriteRules,
 		feed.Crawler,
+		feed.UserAgent,
 		feed.Username,
 		feed.Password,
 		feed.ID,

template/html/add_subscription.html

@@ -41,6 +41,9 @@
 
             <label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "Fetch original content" }}</label>
 
+	        <label for="form-user-agent">{{ t "User-Agent" }}</label>
+	        <input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
+
             <label for="form-feed-username">{{ t "Feed Username" }}</label>
             <input type="text" name="feed_username" id="form-feed-username" value="{{ .form.Username }}">
 

template/html/choose_subscription.html

@@ -19,6 +19,7 @@
 <form action="{{ route "chooseSubscription" }}" method="POST">
     <input type="hidden" name="csrf" value="{{ .csrf }}">
     <input type="hidden" name="category_id" value="{{ .form.CategoryID }}">
+    <input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
     <input type="hidden" name="feed_username" value="{{ .form.Username }}">
     <input type="hidden" name="feed_password" value="{{ .form.Password }}">
     {{ if .form.Crawler }}

template/html/edit_feed.html

@@ -58,6 +58,9 @@
         -->
         <input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
 
+	    <label for="form-user-agent">{{ t "User-Agent" }}</label>
+	    <input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
+
         <label for="form-scraper-rules">{{ t "Scraper Rules" }}</label>
         <input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
 

template/views.go

@@ -87,6 +87,9 @@ var templateViewsMap = map[string]string{
 
             <label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "Fetch original content" }}</label>
 
+	        <label for="form-user-agent">{{ t "User-Agent" }}</label>
+	        <input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
+
             <label for="form-feed-username">{{ t "Feed Username" }}</label>
             <input type="text" name="feed_username" id="form-feed-username" value="{{ .form.Username }}">
 
@@ -265,6 +268,7 @@ var templateViewsMap = map[string]string{
 <form action="{{ route "chooseSubscription" }}" method="POST">
     <input type="hidden" name="csrf" value="{{ .csrf }}">
     <input type="hidden" name="category_id" value="{{ .form.CategoryID }}">
+    <input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
     <input type="hidden" name="feed_username" value="{{ .form.Username }}">
     <input type="hidden" name="feed_password" value="{{ .form.Password }}">
     {{ if .form.Crawler }}
@@ -453,6 +457,9 @@ var templateViewsMap = map[string]string{
         -->
         <input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
 
+	    <label for="form-user-agent">{{ t "User-Agent" }}</label>
+	    <input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
+
         <label for="form-scraper-rules">{{ t "Scraper Rules" }}</label>
         <input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
 
@@ -1359,15 +1366,15 @@ var templateViewsMap = map[string]string{
 
 var templateViewsMapChecksums = map[string]string{
 	"about":               "ad2fb778fc73c39b733b3f81b13e5c7d689b041fadd24ee2d4577f545aa788ad",
-	"add_subscription":    "3fbcffefc94fb0fccfcf870d602f5ba78ce3ab7ebaeacd04198a6e529143cb29",
+	"add_subscription":    "dc6593913e6ff14a6f957349c2ce43f0e3e095e1985f9d739929a88f5f6550bd",
 	"bookmark_entries":    "49423f84c05d77368e20c8e14c53ad237308cdaf4143413487d1b0e11c18d148",
 	"categories":          "ca1280cd157bb527d4fc907da67b05a8347378f6dce965b9389d4bcdf3600a11",
 	"category_entries":    "d219d4bd5376c526c00a3da49b511fb73e812be5d1e12acadeceee8dfa4bbfe2",
-	"choose_subscription": "7266b269ddbe145e757a24a57f3fbc7611e34a20383fbd887988204cebce2681",
+	"choose_subscription": "57280e5b6ba3118c14ef1238e8003f5e92401a671dd6e6e198c864ebbd0b8030",
 	"create_category":     "2b82af5d2dcd67898dc5daa57a6461e6ff8121a6089b2a2a1be909f35e4a2275",
 	"create_user":         "1ef0a1f9bf119d44929c81f13073a257d69650cf5064960cf06a63fe51923e86",
 	"edit_category":       "cee720faadcec58289b707ad30af623d2ee66c1ce23a732965463250d7ff41c5",
-	"edit_feed":           "1a8e342e4fac80e8b9c73537c7fe8aaf7f9e3e7af22f411927010897dd37e9c3",
+	"edit_feed":           "6ff765208828e7533f865009de26edd87d8109b547bdff08132d862aea0e223b",
 	"edit_user":           "7373e09f805e6c017167001519b9feb04226be6c81c2875cbacd5ce94f2c24bf",
 	"entry":               "82a0a4e715da94b12370b380072f1175c9f0e07b37e7f54a9adca4ed1fe015c0",
 	"feed_entries":        "bebc42317ca9e908fcdb98cc1c4a2dc3f4bb7ef6d4c288d3d3fba8f8339403b6",

tests/feed_test.go

@@ -195,6 +195,31 @@ func TestUpdateFeedRewriteRules(t *testing.T) {
 	}
 }
 
+func TestUpdateFeedUserAgent(t *testing.T) {
+	client := createClient(t)
+	feed, _ := createFeed(t, client)
+
+	userAgent := "test"
+	updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModification{UserAgent: &userAgent})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if updatedFeed.UserAgent != userAgent {
+		t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent)
+	}
+
+	userAgent = ""
+	updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModification{UserAgent: &userAgent})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if updatedFeed.UserAgent != userAgent {
+		t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent)
+	}
+}
+
 func TestUpdateFeedUsername(t *testing.T) {
 	client := createClient(t)
 	feed, _ := createFeed(t, client)

ui/entry_scraper.go

@@ -38,7 +38,7 @@ func (c *Controller) FetchContent(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules)
+	content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules, entry.Feed.UserAgent)
 	if err != nil {
 		json.ServerError(w, err)
 		return

ui/feed_edit.go

@@ -7,6 +7,7 @@ package ui  // import "miniflux.app/ui"
 import (
 	"net/http"
 
+	"miniflux.app/http/client"
 	"miniflux.app/http/request"
 	"miniflux.app/http/response/html"
 	"miniflux.app/ui/form"
@@ -52,6 +53,7 @@ func (c *Controller) EditFeed(w http.ResponseWriter, r *http.Request) {
 		ScraperRules: feed.ScraperRules,
 		RewriteRules: feed.RewriteRules,
 		Crawler:      feed.Crawler,
+		UserAgent:    feed.UserAgent,
 		CategoryID:   feed.Category.ID,
 		Username:     feed.Username,
 		Password:     feed.Password,
@@ -66,6 +68,7 @@ func (c *Controller) EditFeed(w http.ResponseWriter, r *http.Request) {
 	view.Set("user", user)
 	view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
 	view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
+	view.Set("defaultUserAgent", client.DefaultUserAgent)
 
 	html.OK(w, r, view.Render("edit_feed"))
 }

ui/feed_update.go

@@ -7,6 +7,7 @@ package ui  // import "miniflux.app/ui"
 import (
 	"net/http"
 
+	"miniflux.app/http/client"
 	"miniflux.app/http/request"
 	"miniflux.app/http/response"
 	"miniflux.app/http/response/html"
@@ -59,6 +60,7 @@ func (c *Controller) UpdateFeed(w http.ResponseWriter, r *http.Request) {
 	view.Set("user", user)
 	view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
 	view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
+	view.Set("defaultUserAgent", client.DefaultUserAgent)
 
 	if err := feedForm.ValidateModification(); err != nil {
 		view.Set("errorMessage", err.Error())

ui/form/feed.go

@@ -20,6 +20,7 @@ type FeedForm struct {
 	ScraperRules string
 	RewriteRules string
 	Crawler      bool
+	UserAgent    string
 	CategoryID   int64
 	Username     string
 	Password     string
@@ -42,6 +43,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
 	feed.ScraperRules = f.ScraperRules
 	feed.RewriteRules = f.RewriteRules
 	feed.Crawler = f.Crawler
+	feed.UserAgent = f.UserAgent
 	feed.ParsingErrorCount = 0
 	feed.ParsingErrorMsg = ""
 	feed.Username = f.Username
@@ -61,6 +63,7 @@ func NewFeedForm(r *http.Request) *FeedForm {
 		SiteURL:      r.FormValue("site_url"),
 		Title:        r.FormValue("title"),
 		ScraperRules: r.FormValue("scraper_rules"),
+		UserAgent:    r.FormValue("user_agent"),
 		RewriteRules: r.FormValue("rewrite_rules"),
 		Crawler:      r.FormValue("crawler") == "1",
 		CategoryID:   int64(categoryID),

ui/form/subscription.go

@@ -16,6 +16,7 @@ type SubscriptionForm struct {
 	URL        string
 	CategoryID int64
 	Crawler    bool
+	UserAgent  string
 	Username   string
 	Password   string
 }
@@ -40,6 +41,7 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
 		URL:        r.FormValue("url"),
 		Crawler:    r.FormValue("crawler") == "1",
 		CategoryID: int64(categoryID),
+		UserAgent:  r.FormValue("user_agent"),
 		Username:   r.FormValue("feed_username"),
 		Password:   r.FormValue("feed_password"),
 	}

ui/subscription_add.go

@@ -7,6 +7,7 @@ package ui  // import "miniflux.app/ui"
 import (
 	"net/http"
 
+	"miniflux.app/http/client"
 	"miniflux.app/http/response/html"
 	"miniflux.app/http/request"
 	"miniflux.app/ui/session"
@@ -35,6 +36,7 @@ func (c *Controller) AddSubscription(w http.ResponseWriter, r *http.Request) {
 	view.Set("user", user)
 	view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
 	view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
+	view.Set("defaultUserAgent", client.DefaultUserAgent)
 
 	html.OK(w, r, view.Render("add_subscription"))
 }

ui/subscription_bookmarklet.go

@@ -7,6 +7,7 @@ package ui  // import "miniflux.app/ui"
 import (
 	"net/http"
 
+	"miniflux.app/http/client"
 	"miniflux.app/http/request"
 	"miniflux.app/http/response/html"
 	"miniflux.app/ui/form"
@@ -39,6 +40,7 @@ func (c *Controller) Bookmarklet(w http.ResponseWriter, r *http.Request) {
 	view.Set("user", user)
 	view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
 	view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
+	view.Set("defaultUserAgent", client.DefaultUserAgent)
 
 	html.OK(w, r, view.Render("add_subscription"))
 }

ui/subscription_choose.go

@@ -7,6 +7,7 @@ package ui  // import "miniflux.app/ui"
 import (
 	"net/http"
 
+	"miniflux.app/http/client"
 	"miniflux.app/http/response"
 	"miniflux.app/http/response/html"
 	"miniflux.app/http/request"
@@ -38,6 +39,7 @@ func (c *Controller) ChooseSubscription(w http.ResponseWriter, r *http.Request)
 	view.Set("user", user)
 	view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
 	view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
+	view.Set("defaultUserAgent", client.DefaultUserAgent)
 
 	subscriptionForm := form.NewSubscriptionForm(r)
 	if err := subscriptionForm.Validate(); err != nil {
@@ -52,6 +54,7 @@ func (c *Controller) ChooseSubscription(w http.ResponseWriter, r *http.Request)
 		subscriptionForm.CategoryID,
 		subscriptionForm.URL,
 		subscriptionForm.Crawler,
+		subscriptionForm.UserAgent,
 		subscriptionForm.Username,
 		subscriptionForm.Password,
 	)

ui/subscription_submit.go

@@ -7,6 +7,7 @@ package ui  // import "miniflux.app/ui"
 import (
 	"net/http"
 
+	"miniflux.app/http/client"
 	"miniflux.app/http/response"
 	"miniflux.app/http/response/html"
 	"miniflux.app/http/request"
@@ -40,6 +41,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request)
 	v.Set("user", user)
 	v.Set("countUnread", c.store.CountUnreadEntries(user.ID))
 	v.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
+	v.Set("defaultUserAgent", client.DefaultUserAgent)
 
 	subscriptionForm := form.NewSubscriptionForm(r)
 	if err := subscriptionForm.Validate(); err != nil {
@@ -51,6 +53,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request)
 
 	subscriptions, err := subscription.FindSubscriptions(
 		subscriptionForm.URL,
+		subscriptionForm.UserAgent,
 		subscriptionForm.Username,
 		subscriptionForm.Password,
 	)
@@ -76,6 +79,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request)
 			subscriptionForm.CategoryID,
 			subscriptions[0].URL,
 			subscriptionForm.Crawler,
+			subscriptionForm.UserAgent,
 			subscriptionForm.Username,
 			subscriptionForm.Password,
 		)