Nested paragraph content.
Nested span.Main content here.
This is the main content.
tags")
+ }
+ if !foundSpanContent {
+ t.Error("Span content should be wrapped in
tags")
+ }
+ if !foundSectionContent {
+ t.Error("Section content should be wrapped in
tags")
+ }
+
+ // Verify overall structure
+ if !strings.HasPrefix(result, "
") || !strings.HasSuffix(result, "
") {
+ t.Error("Result should be wrapped in outer tags")
+ }
+}
+
+func TestGetArticleEmptyAndEdgeCases(t *testing.T) {
+ testCases := []struct {
+ name string
+ html string
+ expected string
+ }{
+ {
+ name: "empty body",
+ html: ``,
+ expected: ` `,
+ expected: ` `, // body is top candidate, includes inner div
+ },
+ {
+ name: "self-closing elements",
+ html: `
`,
+ expected: `
`, // body includes inner div
+ },
+ {
+ name: "nested structure with no text",
+ html: ``,
+ expected: ``, // body includes inner div
+ },
+ {
+ name: "complex nesting with mixed content",
+ html: ``,
+ expected: ``, // The #main div gets selected as top candidate, not body
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.html))
+ if err != nil {
+ t.Fatalf("Failed to parse HTML: %v", err)
+ }
+
+ candidates := getCandidates(doc)
+ topCandidate := getTopCandidate(doc, candidates)
+
+ result := getArticle(topCandidate, candidates)
+
+ if result != tc.expected {
+ t.Errorf("\nExpected:\n%s\n\nGot:\n%s", tc.expected, result)
+ }
+ })
+ }
+}
+
+// Test helper functions used by getArticle
+func TestGetLinkDensity(t *testing.T) {
+ testCases := []struct {
+ name string
+ html string
+ expected float32
+ }{
+ {
+ name: "no links",
+ html: ` `,
+ expected: 0.0,
+ },
+ {
+ name: "links with no text",
+ html: ``,
+ expected: 0.0, // Empty link contributes 0 to link length
+ },
+ {
+ name: "multiple links",
+ html: ``,
+ expected: float32(11) / float32(29), // "firstsecond" vs "Start first middle second end"
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.html))
+ if err != nil {
+ t.Fatalf("Failed to parse HTML: %v", err)
+ }
+
+ selection := doc.Find("div").First()
+ result := getLinkDensity(selection)
+
+ // Use a small epsilon for float comparison
+ epsilon := float32(0.001)
+ if result < tc.expected-epsilon || result > tc.expected+epsilon {
+ t.Errorf("Expected link density %f, got %f", tc.expected, result)
+ }
+ })
+ }
+}
+
+func TestContainsSentence(t *testing.T) {
+ testCases := []struct {
+ name string
+ content string
+ expected bool
+ }{
+ {
+ name: "ends with period",
+ content: "This is a sentence.",
+ expected: true,
+ },
+ {
+ name: "contains period with space",
+ content: "First sentence. Second sentence",
+ expected: true,
+ },
+ {
+ name: "no sentence markers",
+ content: "Just a fragment",
+ expected: false,
+ },
+ {
+ name: "period without space",
+ content: "Something.else",
+ expected: false,
+ },
+ {
+ name: "empty string",
+ content: "",
+ expected: false,
+ },
+ {
+ name: "only period",
+ content: ".",
+ expected: true,
+ },
+ {
+ name: "period and space at end",
+ content: "Sentence. ",
+ expected: true,
+ },
+ {
+ name: "multiple sentences",
+ content: "First. Second. Third",
+ expected: true,
+ },
+ {
+ name: "period in middle only",
+ content: "Text. More text",
+ expected: true,
+ },
+ {
+ name: "whitespace around period",
+ content: "Text . More",
+ expected: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ result := containsSentence(tc.content)
+ if result != tc.expected {
+ t.Errorf("Expected %v for content %q, got %v", tc.expected, tc.content, result)
+ }
+ })
+ }
+}
`, // getTopCandidate returns body, body has no inner HTML
+ },
+ {
+ name: "only whitespace content",
+ html: `
Content
Content
Nested content
Paragraph in nested structure.
Nested content
Paragraph in nested structure.
This is plain text content with no links at all.
`,
+ expected: 0.0,
+ },
+ {
+ name: "all links",
+ html: ``,
+ expected: 1.0,
+ },
+ {
+ name: "half links",
+ html: `Plain text Link text
`,
+ expected: 0.45, // "Link text" is 9 chars, "Plain text Link text" is 20 chars
+ },
+ {
+ name: "nested links",
+ html: `Text Link nested more text
`,
+ expected: float32(11) / float32(26), // "Link nested" vs "Text Link nested more text"
+ },
+ {
+ name: "empty content",
+ html: ``,
+ expected: 0.0,
+ },
+ {
+ name: "whitespace only",
+ html: `