tags")
}
if !foundSpanContent {
t.Error("Span content should be wrapped in
tags")
}
if !foundSectionContent {
t.Error("Section content should be wrapped in
tags")
}
// Verify overall structure
if !strings.HasPrefix(result, "
") || !strings.HasSuffix(result, "
") {
t.Error("Result should be wrapped in outer tags")
}
}
func TestGetArticleEmptyAndEdgeCases(t *testing.T) {
testCases := []struct {
name string
html string
expected string
}{
{
name: "empty body",
html: ``,
expected: ` `,
expected: ` `, // body is top candidate, includes inner div
},
{
name: "self-closing elements",
html: `
`,
expected: `
`, // body includes inner div
},
{
name: "nested structure with no text",
html: ``,
expected: ``, // body includes inner div
},
{
name: "complex nesting with mixed content",
html: ``,
expected: ``, // The #main div gets selected as top candidate, not body
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.html))
if err != nil {
t.Fatalf("Failed to parse HTML: %v", err)
}
candidates := getCandidates(doc)
topCandidate := getTopCandidate(doc, candidates)
result := getArticle(topCandidate, candidates)
if result != tc.expected {
t.Errorf("\nExpected:\n%s\n\nGot:\n%s", tc.expected, result)
}
})
}
}
// Test helper functions used by getArticle
func TestGetLinkDensity(t *testing.T) {
testCases := []struct {
name string
html string
expected float32
}{
{
name: "no links",
html: ` `,
expected: 0.0,
},
{
name: "links with no text",
html: ``,
expected: 0.0, // Empty link contributes 0 to link length
},
{
name: "multiple links",
html: ``,
expected: float32(11) / float32(29), // "firstsecond" vs "Start first middle second end"
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.html))
if err != nil {
t.Fatalf("Failed to parse HTML: %v", err)
}
selection := doc.Find("div").First()
result := getLinkDensity(selection)
// Use a small epsilon for float comparison
epsilon := float32(0.001)
if result < tc.expected-epsilon || result > tc.expected+epsilon {
t.Errorf("Expected link density %f, got %f", tc.expected, result)
}
})
}
}
func TestContainsSentence(t *testing.T) {
testCases := []struct {
name string
content string
expected bool
}{
{
name: "ends with period",
content: "This is a sentence.",
expected: true,
},
{
name: "contains period with space",
content: "First sentence. Second sentence",
expected: true,
},
{
name: "no sentence markers",
content: "Just a fragment",
expected: false,
},
{
name: "period without space",
content: "Something.else",
expected: false,
},
{
name: "empty string",
content: "",
expected: false,
},
{
name: "only period",
content: ".",
expected: true,
},
{
name: "period and space at end",
content: "Sentence. ",
expected: true,
},
{
name: "multiple sentences",
content: "First. Second. Third",
expected: true,
},
{
name: "period in middle only",
content: "Text. More text",
expected: true,
},
{
name: "whitespace around period",
content: "Text . More",
expected: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := containsSentence(tc.content)
if result != tc.expected {
t.Errorf("Expected %v for content %q, got %v", tc.expected, tc.content, result)
}
})
}
}
`, // getTopCandidate returns body, body has no inner HTML
},
{
name: "only whitespace content",
html: `
Content
Content
Nested content
Paragraph in nested structure.
Nested content
Paragraph in nested structure.
This is plain text content with no links at all.
`,
expected: 0.0,
},
{
name: "all links",
html: ``,
expected: 1.0,
},
{
name: "half links",
html: `Plain text Link text
`,
expected: 0.45, // "Link text" is 9 chars, "Plain text Link text" is 20 chars
},
{
name: "nested links",
html: `Text Link nested more text
`,
expected: float32(11) / float32(26), // "Link nested" vs "Text Link nested more text"
},
{
name: "empty content",
html: ``,
expected: 0.0,
},
{
name: "whitespace only",
html: `