Skip to content

Commit 032351e

Browse files
committed
Added webpage
1 parent 74dde91 commit 032351e

File tree

5 files changed

+205
-45
lines changed

5 files changed

+205
-45
lines changed

.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -168,3 +168,6 @@ temp/
168168
Network Trash Folder
169169
Temporary Items
170170
.apdisk
171+
172+
*.fiber.gz
173+
.vscode

Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:alpine AS builder
1+
FROM golang:1.16-alpine AS builder
22
RUN apk update && apk add --no-cache git && apk add -U --no-cache ca-certificates
33
WORKDIR /app/
44
ADD go.mod go.sum ./

go.mod

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
module feed-fetcher
22

3-
go 1.15
3+
go 1.16
44

55
require (
6+
github.com/PuerkitoBio/goquery v1.5.1
67
github.com/gofiber/fiber/v2 v2.6.0
78
github.com/mmcdole/gofeed v1.1.0
89
)

index.html

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<title>feed-fetcher</title>
5+
6+
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml, %3Csvg xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' viewBox='0 0 512 512' style='height:50px;'%3E%3Ccircle cx='256' cy='256' r='247.9' fill='%2371cad1'/%3E%3Cg%3E%3Cpath d='M159 397a44 44 0 110-88 44 44 0 010 88z' fill='%23ef71a8'/%3E%3Cpath d='M250 384v-10c0-62-50-112-112-112h-10l-2-2v-56l2-2h10c95 0 172 77 172 172v10l-2 2h-56l-2-2z' fill='%23ef71a8'/%3E%3Cpath d='M358 384v-10a220 220 0 00-220-220h-10l-2-2V96l2-2h12c74 0 144 29 197 81a276 276 0 0181 197v12l-2 2h-56l-2-2z' fill='%23ef71a8'/%3E%3C/g%3E%3Cpath d='M437 75a254 254 0 00-362 0 255 255 0 000 362 255 255 0 00362 0 255 255 0 000-362zM256 496a240 240 0 111-480 240 240 0 01-1 480z' fill='%2327162a'/%3E%3C/svg%3E">
7+
8+
9+
<meta property="og:title" content="feed-fetcher">
10+
<meta property="og:site_name" content="feed-fetcher">
11+
<meta property="og:url" content="https://feed-fetcher.cluster.fun">
12+
<meta property="og:description" content="Returns the RSS feed associated with the given URL">
13+
<meta property="og:type" content="website">
14+
<meta property="og:image" content="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB2aWV3Qm94PSItNTAwIC0xMDAgMTYxMiA3MTIiIHdpZHRoPSIxMDAwIiBoZWlnaHQ9IjEwMDAiPjxjaXJjbGUgY3g9IjI1NiIgY3k9IjI1NiIgcj0iMjQ3LjkiIGZpbGw9IiM3MWNhZDEiLz48Zz48cGF0aCBkPSJNMTU5IDM5N2E0NCA0NCAwIDExMC04OCA0NCA0NCAwIDAxMCA4OHoiIGZpbGw9IiNlZjcxYTgiLz48cGF0aCBkPSJNMjUwIDM4NHYtMTBjMC02Mi01MC0xMTItMTEyLTExMmgtMTBsLTItMnYtNTZsMi0yaDEwYzk1IDAgMTcyIDc3IDE3MiAxNzJ2MTBsLTIgMmgtNTZsLTItMnoiIGZpbGw9IiNlZjcxYTgiLz48cGF0aCBkPSJNMzU4IDM4NHYtMTBhMjIwIDIyMCAwIDAwLTIyMC0yMjBoLTEwbC0yLTJWOTZsMi0yaDEyYzc0IDAgMTQ0IDI5IDE5NyA4MWEyNzYgMjc2IDAgMDE4MSAxOTd2MTJsLTIgMmgtNTZsLTItMnoiIGZpbGw9IiNlZjcxYTgiLz48L2c+PHBhdGggZD0iTTQzNyA3NWEyNTQgMjU0IDAgMDAtMzYyIDAgMjU1IDI1NSAwIDAwMCAzNjIgMjU1IDI1NSAwIDAwMzYyIDAgMjU1IDI1NSAwIDAwMC0zNjJ6TTI1NiA0OTZhMjQwIDI0MCAwIDExMS00ODAgMjQwIDI0MCAwIDAxLTEgNDgweiIgZmlsbD0iIzI3MTYyYSIvPjwvc3ZnPgo=">
15+
<meta name="twitter:card" content="summary" />
16+
<meta name="twitter:creator" content="@Marcus_Noble_" />
17+
18+
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300italic,700,700italic">
19+
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/normalize/5.0.0/normalize.css">
20+
<link rel="stylesheet" href="https://cdn.githubraw.com/AverageMarcus/milligram/fb7f0444/dist/milligram.min.css">
21+
<style>
22+
body {
23+
min-height: 100vh;
24+
display: flex;
25+
flex-direction: column;
26+
justify-content: space-between;
27+
}
28+
</style>
29+
</head>
30+
<body>
31+
<div class="container">
32+
<h1 class="heading-fancy">
33+
feed-fetcher
34+
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 512 512" style="height:50px;"><circle cx="256" cy="256" r="247.9" fill="#71cad1"/><g><path d="M159 397a44 44 0 110-88 44 44 0 010 88z" fill="#ef71a8"/><path d="M250 384v-10c0-62-50-112-112-112h-10l-2-2v-56l2-2h10c95 0 172 77 172 172v10l-2 2h-56l-2-2z" fill="#ef71a8"/><path d="M358 384v-10a220 220 0 00-220-220h-10l-2-2V96l2-2h12c74 0 144 29 197 81a276 276 0 0181 197v12l-2 2h-56l-2-2z" fill="#ef71a8"/></g><path d="M437 75a254 254 0 00-362 0 255 255 0 000 362 255 255 0 00362 0 255 255 0 000-362zM256 496a240 240 0 111-480 240 240 0 01-1 480z" fill="#27162a"/></svg>
35+
</h1>
36+
<blockquote>Returns the RSS feed associated with the given URL</blockquote>
37+
38+
<p>
39+
Enter an URL and press "Fetch Feed" to see all feeds (if any) found associated with that page.
40+
</p>
41+
42+
<div class="row">
43+
<form id="fetchForm" class="column column-80 column-offset-10" method="GET" action="/">
44+
<fieldset>
45+
<label for="url">URL of webpage</label>
46+
<input type="text" placeholder="e.g. https://marcusnoble.co.uk" name="url" id="url">
47+
<input id="fetchFeed" class="button-primary button-outline float-right" type="submit" value="Fetch Feed">
48+
</fieldset>
49+
</form>
50+
</div>
51+
<div class="row">
52+
<div id="results" class="column column-80 column-offset-10 text-center"></div>
53+
</div>
54+
55+
<hr>
56+
57+
<p>
58+
<em>Alternatively</em>, you can navigate to <code>https://feed-fetcher.cluster.fun/?url=YOUR_URL_HERE</code> and your browser will redirect to the associated feed URL if found.
59+
</p>
60+
<p>
61+
<h4>Calling as an API:</h4>
62+
If you set the <code>Content-Type</code> request header to <code>application/json</code> the response will return as a JSON array of all found feed URLs. If no feeds are found an empty array will be returned and the response status code will be <strong>404</strong>. If multiple feeds are found all will be returned in the array with a response status code of <strong>300</strong>.
63+
<pre><code>
64+
✨ curl -H "Content-Type: application/json" https://feed-fetcher.cluster.fun/\?url\=https://marcusnoble.co.uk
65+
HTTP/1.1 200 OK
66+
Date: Sun, 21 Mar 2021 07:24:37 GMT
67+
Content-Type: application/json
68+
Content-Length: 38
69+
70+
[
71+
"https://marcusnoble.co.uk/feed.xml"
72+
]
73+
</code></pre>
74+
</p>
75+
76+
<div>
77+
Source code available on <a href="https://github.com/AverageMarcus/feed-fetcher" target="_blank" rel="noopener noreferrer">GitHub</a>, <a href="https://gitlab.com/AverageMarcus/feed-fetcher" target="_blank" rel="noopener noreferrer">GitLab</a>, <a href="https://bitbucket.org/AverageMarcus/feed-fetcher/" target="_blank" rel="noopener noreferrer">Bitbucket</a> & <a href="https://git.cluster.fun/AverageMarcus/feed-fetcher" target="_blank" rel="noopener noreferrer">my own Gitea server</a>.
78+
</div>
79+
</div>
80+
81+
<div class="container">
82+
<div class="row">
83+
<div class="column column-60 column-offset-20">
84+
<footer>
85+
Made with
86+
<svg height="20" class="fill-primary" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 449.3 449.3" xmlns:xlink="http://www.w3.org/1999/xlink"><title>love</title><g><path d="M0 162.7c1.5-7.7 2.7-15.4 4.5-23A125.5 125.5 0 0132 88a136.3 136.3 0 0162.7-40.6c8.3-2.9 17.7-3.2 26.6-3.7a134 134 0 0155.6 6.6c14.9 5.7 30 11 41 23.6 17-20 36.4-36.3 60-46.4 12-5.2 25.7-6.9 38.7-9.4a79.4 79.4 0 0140.3 3.2 96.4 96.4 0 0143.2 26 209.8 209.8 0 0137.8 55.4 133.2 133.2 0 0111 65.7c-3.2 42.2-21 79-41.5 114.8a431.2 431.2 0 01-47.6 64.3c-19.6 23-39.7 45.7-59.6 68.5-3.7 4.3-7.2 9-11.7 12.4-7.3 5.4-15.9 4.9-23.8 1.5-21.9-9.2-43.8-18.5-65.3-28.5a520.1 520.1 0 01-98-58.7c-28.2-21.5-55.5-44.3-74-75.3a183.8 183.8 0 01-25-61.4c-1-6.2-1.6-12.6-2.4-18.8v-24.5zM138 281l2.5.1c0 2.1.3 4.3 0 6.3l-9 55.5c-.3 1.6 0 4.2 1 5 5.8 4.1 11.8 7.8 17.9 11.8l11-59.7 2.5.3c3.7 21.6-6.3 42-7.6 63.5l19.4 9.6c.5-18 2.2-35 6.9-51.6 1.3 4 1.5 8 1.2 12-.7 11.2-1 22.5-2.6 33.7-1 7.3 1.9 10.6 8 13.3 27 11.7 54.1 23.5 81 35.7 5.5 2.5 8.8 1.5 12.6-2.9 12.5-14.4 25.7-28.1 38-42.7 17.2-20.2 34.5-40.5 50.5-61.7 29.5-39.2 52.7-81.7 61-131 3.6-21.7 2-43-5.6-63.5a176 176 0 00-33.9-54.5 84.8 84.8 0 00-38-26 91.2 91.2 0 00-44.2-2.5c-13 2-25.6 5.1-37.2 12.3a208.6 208.6 0 00-45.9 37c-5.7 6.3-7.8 6.4-15 1.4-5.7-4-11-8.8-17.2-11.9-15.1-7.5-30.7-14.4-48-14.9-13.7-.4-28-2-41.2 1-36 8.4-62.7 30-80.3 62.8a111 111 0 00-11.7 56.6c.3 10 1.4 20 2.2 29.9 7-18.9 11-38.3 19.7-56.3.6 3 .6 6 0 8.7l-14.5 53.2c-.7 2.6-2 5.7-1.2 8 2.7 8.4 6.2 16.6 9.4 24.8 0-20 13.7-63.9 21.8-66.8 0 .6.3 1.1.2 1.6a11936 11936 0 01-17 64c-.3 1.4-1.6 2.6-2.9 4.5l9.7 17a573 573 0 0120-67.3l2.9.7c-1 5-1.5 10-2.8 14.9-4.7 17.2-9.6 34.3-14.2 51.5-.7 2.5-1.5 6-.3 7.8 3.6 5.6 8 10.7 12.4 16.1 1.8-8.6 3.1-16.6 5.2-24.4 3.1-11.5 6.6-22.9 10.2-34.3.8-2.6 2.7-5 4-7.4l2 .8c-.1 1.6-.1 3.4-.5 5l-12.2 47.4c-4.6 17.9-4.3 18.9 7.8 29.4 2.2 1.8 4.5 3.5 7 5.3 3.7-31 12.5-64.7 18.4-68-.3 3.5-.3 6.6-.9 9.6l-12.7 58.8c-.3 1.2-.7 3-.1 3.5 4.7 4.4 9.6 8.6 14.5 12.9 3.5-21.4 7.6-41.7 13.6-61.6 3.4 8.7.6 17-.8 25.2-1.9 11.6-4.5 23.1-6.5 34.7-.4 2.3-.4 6 1 7 4.8 4.2 10.3 7.4 16 11.3 3.9-21.5 5.5-42.6 12.6-62.5z"/><path d="M323.2 180.5a24 24 0 01-24.5-19.7c-2-9.7.7-20.2 15-27 16.6-7.8 38.3 2.3 41.6 19.5 1.2 6.6-1.6 12.1-6 16.3a35.1 35.1 0 01-26 11z"/><path d="M138.9 167.1a31 31 0 0125 12.7 22 22 0 01-13.6 34.9 29.9 29.9 0 01-31-9.9c-6-7-7.6-15.3-4.1-24.1 3.7-9.5 12-12 21-13.5.8-.2 1.8 0 2.7 0z"/><path d="M233.2 202c-18.5-.4-33.7-13.6-34-29.7 0-4.2.4-8 5.4-8.7 4.5-.7 6.6 2.3 8 6 2.3 6.6 5.5 12.4 12.4 15.4 7.5 3.2 14 1.5 20.4-3.3 6.2-4.7 6.4-11 4.7-17.5-1.4-5.2.4-8.4 4.7-10.2 4.3-1.7 9 1.3 10.9 6.2 7.2 19.8-5.7 34.6-22.8 40.2-3 1-6.5 1-9.7 1.6z"/><path d="M201.2 384.7c-.9-2-2.7-4.2-2.5-6.2 1.2-14 2.8-28.1 4.4-42.2 0-.9.7-1.7 1-2.6l2.6 1.1-3.1 49.1-2.4.8z"/><path d="M222 387.4c-4.8-5.7-5-15-1-37.8 3.1 3 4 30.5 1 37.8z"/><path d="M240.9 361.7l-1.4 20.2c-4.8-3.3-4.6-11.9-.2-20.4l1.6.2z"/><path d="M257.4 380.4c-4.1-5.2-3.7-9.7 1-14.7l-1 14.7z"/></g></svg>
87+
by <a href="https://marcusnoble.co.uk" class="fancy-link">Marcus Noble</a>
88+
</footer>
89+
</div>
90+
</div>
91+
</div>
92+
93+
<script>
94+
document.getElementById('fetchForm').addEventListener('submit', function(e) {
95+
let submitUrl = e.target.action + '?url=' + document.getElementById('url').value;
96+
97+
let resultsText = document.getElementById('results');
98+
fetch(submitUrl, { headers: { "Content-Type": "application/json" } })
99+
.then(response => {
100+
if (response.status == 404) {
101+
resultsText.innerHTML = "No feeds found";
102+
return;
103+
} else if (response.status > 400) {
104+
resultsText.innerHTML = "Unable to fetch feed";
105+
return;
106+
}
107+
108+
return response.json()
109+
})
110+
.then(json => {
111+
if (json) {
112+
resultsText.innerHTML = `<h3 class="heading-fancy">Results</h3><br><ul>` + json.reduce((acc, url) => `${acc}<li><a href="${url}" class="fancy-link">${url}</a></li>`, '') + '</ul>';
113+
}
114+
})
115+
.catch(ex => {
116+
console.log(ex);
117+
resultsText.innerHTML = "Unable to fetch feed";
118+
});
119+
120+
e.preventDefault();
121+
});
122+
</script>
123+
</body>
124+
</html>

main.go

+75-43
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,18 @@ import (
77
"os"
88
"strings"
99

10+
"embed"
11+
1012
"github.com/PuerkitoBio/goquery"
1113
"github.com/gofiber/fiber/v2"
1214
"github.com/mmcdole/gofeed"
1315
)
1416

17+
//go:embed index.html
18+
19+
var content embed.FS
20+
1521
func main() {
16-
fp := gofeed.NewParser()
1722
port, ok := os.LookupEnv("PORT")
1823
if !ok {
1924
port = "8080"
@@ -24,56 +29,32 @@ func main() {
2429
app.Get("/", func(c *fiber.Ctx) error {
2530
feedUrl := c.Query("url")
2631
if feedUrl == "" {
27-
fmt.Println("No URL provided")
28-
return c.SendStatus(fiber.StatusBadRequest)
32+
c.Type("html", "UTF8")
33+
body, _ := content.ReadFile("index.html")
34+
return c.Send(body)
2935
}
3036

31-
_, err := fp.ParseURL(feedUrl)
32-
if err != nil && err == gofeed.ErrFeedTypeNotDetected {
33-
res, err := http.Get(feedUrl)
34-
if err != nil {
35-
fmt.Println("Failed to fetch URL")
36-
return c.SendStatus(fiber.StatusInternalServerError)
37-
}
38-
defer res.Body.Close()
39-
if res.StatusCode >= 400 {
40-
fmt.Println("Provided URL returned an error status code")
41-
return c.SendStatus(res.StatusCode)
42-
}
37+
feeds, statusCode := getFeeds(feedUrl)
4338

44-
doc, err := goquery.NewDocumentFromReader(res.Body)
45-
if err != nil {
46-
fmt.Println("Failed to parse response body")
47-
return c.SendStatus(fiber.StatusInternalServerError)
39+
if c.Is("json") {
40+
if statusCode >= 400 || statusCode == 300 {
41+
c.Status(statusCode)
4842
}
43+
return c.JSON(feeds)
44+
} else {
45+
c.Status(statusCode)
46+
c.Location(feeds[0])
4947

50-
matches := doc.Find(`[rel="alternate"][type="application/rss+xml"]`)
51-
if matches.Length() == 0 {
52-
fmt.Println("No RSS feeds found on page")
53-
return c.SendStatus(fiber.StatusNotFound)
48+
if len(feeds) > 1 {
49+
responseBody := "Multiple Choices\n\n"
50+
for _, feed := range feeds {
51+
responseBody += feed + "\n"
52+
}
53+
return c.SendString(responseBody)
5454
}
5555

56-
foundUrl, ok := matches.First().Attr("href")
57-
if !ok {
58-
fmt.Println("href attribute missing from tag")
59-
return c.SendStatus(fiber.StatusNotFound)
60-
}
61-
c.Set("Location", absoluteUrl(feedUrl, foundUrl))
62-
if matches.Length() > 1 {
63-
fmt.Println("Multiple feeds found on page")
64-
return c.SendStatus(fiber.StatusMultipleChoices)
65-
} else {
66-
fmt.Println("Feed found on page")
67-
return c.SendStatus(fiber.StatusTemporaryRedirect)
68-
}
69-
} else if err != nil {
70-
fmt.Println("Failed while attempting to parse feed")
71-
return c.SendStatus(fiber.StatusInternalServerError)
56+
return c.Send(nil)
7257
}
73-
74-
fmt.Println("URL provided is already a feed")
75-
c.Set("Location", feedUrl)
76-
return c.SendStatus(fiber.StatusMovedPermanently)
7758
})
7859

7960
fmt.Println(app.Listen(fmt.Sprintf(":%s", port)))
@@ -87,3 +68,54 @@ func absoluteUrl(requestUrl, foundUrl string) string {
8768

8869
return foundUrl
8970
}
71+
72+
func getFeeds(requestURL string) ([]string, int) {
73+
feeds := []string{}
74+
75+
fp := gofeed.NewParser()
76+
_, err := fp.ParseURL(requestURL)
77+
if err == nil {
78+
feeds = []string{requestURL}
79+
} else if err != nil && err == gofeed.ErrFeedTypeNotDetected {
80+
res, err := http.Get(requestURL)
81+
if err != nil {
82+
fmt.Println("Failed to fetch URL")
83+
return feeds, fiber.StatusInternalServerError
84+
}
85+
defer res.Body.Close()
86+
87+
if res.StatusCode >= 400 {
88+
fmt.Println("Provided URL returned an error status code")
89+
return feeds, res.StatusCode
90+
}
91+
92+
doc, err := goquery.NewDocumentFromReader(res.Body)
93+
if err != nil {
94+
fmt.Println("Failed to parse response body")
95+
return feeds, fiber.StatusInternalServerError
96+
}
97+
98+
matches := doc.Find(`[rel="alternate"][type="application/rss+xml"]`)
99+
if matches.Length() == 0 {
100+
fmt.Println("No RSS feeds found on page")
101+
return feeds, fiber.StatusNotFound
102+
}
103+
104+
matches.Each(func(i int, s *goquery.Selection) {
105+
feeds = append(feeds, absoluteUrl(requestURL, s.AttrOr("href", "")))
106+
})
107+
108+
if matches.Length() > 1 {
109+
fmt.Println("Multiple feeds found on page")
110+
return feeds, fiber.StatusMultipleChoices
111+
} else {
112+
fmt.Println("Feed found on page")
113+
return feeds, fiber.StatusTemporaryRedirect
114+
}
115+
} else if err != nil {
116+
fmt.Println("Failed while attempting to parse feed")
117+
return feeds, fiber.StatusInternalServerError
118+
}
119+
120+
return feeds, 200
121+
}

0 commit comments

Comments
 (0)