use headless

Author: kaki-admin

Dockerfile.backend

@@ -10,10 +10,10 @@ RUN go mod download
 RUN cd /backendBuild/backend-server/ && \
     go build -o backend_cmd  -ldflags="-s -w " main.go 
 
-FROM alpine:latest
-#FROM docker.io/chromedp/headless-shell:latest
-#RUN apt-get update; apt install dumb-init -y; apt install ca-certificates -y
-#ENTRYPOINT ["dumb-init", "--"]
+#FROM alpine:latest
+FROM docker.io/chromedp/headless-shell:latest
+RUN apt-get update; apt install dumb-init -y; apt install ca-certificates -y
+ENTRYPOINT ["dumb-init", "--"]
 
 COPY --from=build /backendBuild/backend-server/backend_cmd /usr/bin/backend_cmd
 

backend-server/crawler/bskyapi/client.go

@@ -61,7 +61,8 @@ func Fetch(websiteURL string) *model.Entry {
 
 func generateEntry(resp *Response) *model.Entry {
 	entry := new(model.Entry)
-	entry.FullContent = "<p>" + resp.Thread.Post.Record.Text + "</p>"
+
+	entry.FullContent = "<p>" + strings.ReplaceAll(resp.Thread.Post.Record.Text, "\n", "<br>") + "</p>"
 	if resp.Thread.Post.Embed.EmbedType == "app.bsky.embed.images#view" {
 		for _, image := range resp.Thread.Post.Embed.EmbedImages {
 			entry.FullContent = entry.FullContent + "<img src='" + image.Thumb + "' /><br>"

backend-server/crawler/threads/client.go

@@ -43,7 +43,7 @@ func ThreadsByheadless(websiteURL string) string {
 	//ctx, cancel := chromedp.NewContext(context.Background())
 	defer cancelCtx()
 	htmlContent := ""
-	common.Logger.Info("notion headless fetch 1 ")
+	common.Logger.Info("threads headless fetch 1 ")
 	err := chromedp.Run(allocCtx,
 		chromedp.Navigate(websiteURL),
 		chromedp.WaitVisible(`[data-pressable-container=true]`, chromedp.ByQuery),