v0.2

- Include comment text in notification
- Include commenter in notification
- Use Timestamp to check if a comment is new
- Set sleep interval between requests in env variable

config/types.go

@@ -6,4 +6,5 @@ type config struct {
 	DebugMode        bool     `env:"DEBUGMODE, default=false"`
 	ChatID           int64    `env:"CHATID"`
 	Watchers         []uint64 `env:"WATCHERS"`
+	SleepInterval    int      `env:"SLEEPINTERVAL"`
 }

docker-compose.yaml

@@ -11,3 +11,4 @@ services:
       - DebugMode=false
       - CHATID=123
       - WATCHERS=123,456,789
+      - SLEEPINTERVAL=60

go.mod

@@ -3,15 +3,17 @@ module watn3y/steamsalty
 go 1.23.4
 
 require (
+	github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1
+	github.com/PuerkitoBio/goquery v1.10.0
 	github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
 	github.com/rs/zerolog v1.33.0
 	github.com/sethvargo/go-envconfig v1.1.0
-	github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1
 )
 
 require (
-	
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 )

go.sum

@@ -1,5 +1,9 @@
 github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1 h1:PD13eMe9XAgPQ0SYWyirqwyOJG90TlEWApCw8A699l0=
 github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1/go.mod h1:eQR7Xf64m2ALDAQE7Nr9ylFZhav1izvF3zzysKPhb0I=
+github.com/PuerkitoBio/goquery v1.10.0 h1:6fiXdLuUvYs2OJSvNRqlNPoBm6YABE226xrbavY5Wv4=
+github.com/PuerkitoBio/goquery v1.10.0/go.mod h1:TjZZl68Q3eGHNBA8CWaxAN7rOU1EbDz3CWuolcO5Yu4=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc=
 github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8=
@@ -18,8 +22,71 @@ github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
 github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog=
 github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

steam/api.go

@@ -12,6 +12,6 @@ func getPlayerDetails(steamID uint64) (summary steamapi.PlayerSummary) {
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to get Player Summary")
 	}
-
+	log.Debug().Interface("Player", response[0]).Msg("Successfully got PlayerSummary from Steam API")
 	return response[0]
 }

steam/http.go

@@ -2,14 +2,18 @@ package steam
 
 import (
 	"encoding/json"
-	"github.com/rs/zerolog/log"
 	"io"
 	"net/http"
 	"net/url"
+	"slices"
 	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/rs/zerolog/log"
 )
 
-func getComments(steamID uint64, start int, count int) (comments CommentResponse) {
+func getComments(steamID uint64, start int, count int) (page CommentsPage) {
 
 	baseURL := "https://steamcommunity.com/comment/Profile/render/"
 
@@ -38,11 +42,49 @@ func getComments(steamID uint64, start int, count int) (comments CommentResponse
 		log.Trace().Interface("Body", resp.Body)
 	}
 
-	err = json.Unmarshal(body, &comments)
+	err = json.Unmarshal(body, &page)
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to parse Comments as JSON")
 	}
 
-	log.Debug().Interface("CommentPage", comments).Msg("Successfully got Comment Page")
+	log.Debug().Interface("CommentPage", page).Uint64("ProfileID", steamID).Msg("Successfully got Comment Page")
+
+	return page
+}
+
+func parseComments(rawComments CommentsPage) (comments []Comment) {
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(rawComments.CommentsHTML))
+	if err != nil {
+		log.Error().Err(err).Msg("Error while parsing CommentsHTML")
+		return
+	}
+	doc.Find(".commentthread_comment.responsive_body_text").Each(func(i int, s *goquery.Selection) {
+		var c Comment
+
+		parsedID, err := strconv.ParseUint(strings.TrimPrefix(s.AttrOr("id", ""), "comment_"), 10, 64)
+		if err != nil {
+			log.Error().Err(err).Msg("Error while parsing Comment ID")
+			return
+		}
+		c.ID = parsedID
+
+		c.Timestamp, err = strconv.ParseInt(s.Find(".commentthread_comment_timestamp").AttrOr("data-timestamp", ""), 10, 64)
+		if err != nil {
+			log.Error().Err(err).Msg("Error while parsing Comment Timestamp")
+			return
+		}
+
+		c.Author = s.Find(".commentthread_comment_author .hoverunderline bdi").Text()
+
+		c.AuthorProfileURL, _ = s.Find(".commentthread_comment_author .hoverunderline").Attr("href")
+
+		c.Text = strings.TrimSpace(s.Find(".commentthread_comment_text").Text())
+
+		comments = append(comments, c)
+	})
+
+	slices.Reverse(comments)
+	log.Debug().Interface("Comments", comments).Msg("Successfully parsed Comment Page")
 	return comments
 }

steam/profile.go

@@ -2,16 +2,21 @@ package steam
 
 import (
 	"fmt"
-	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
-	"github.com/rs/zerolog/log"
 	"math"
+
+	"strings"
 	"sync"
 	"time"
+
 	"watn3y/steamsalty/botIO"
 	"watn3y/steamsalty/config"
+
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
+	"github.com/rs/zerolog/log"
 )
 
-var sleeptime time.Duration = 1 * time.Minute
+var sleeptime time.Duration = time.Duration(config.BotConfig.SleepInterval) * time.Second
+var steamContentCheckText string = "This comment is awaiting analysis by our automated content check system. It will be temporarily hidden until we verify that it does not contain harmful content (e.g. links to websites that attempt to steal information)."
 
 func StartWatchers(bot *tgbotapi.BotAPI) {
 	var wg sync.WaitGroup
@@ -29,29 +34,47 @@ func StartWatchers(bot *tgbotapi.BotAPI) {
 
 func watcher(bot *tgbotapi.BotAPI, steamID uint64) {
 	log.Info().Uint64("SteamID", steamID).Msg("Started Watcher")
-	var previousCount int
+
+	var newestProcessedComment int64 = 0
+
 	for {
-		currentCount := getComments(steamID, math.MaxInt32, 0).TotalCount
+		currentCommentsPage := getComments(steamID, 0, math.MaxInt32)
-		if previousCount == 0 || currentCount <= previousCount {
+		if newestProcessedComment == 0 || newestProcessedComment == currentCommentsPage.TimeLastPost {
-			previousCount = currentCount
+			newestProcessedComment = currentCommentsPage.TimeLastPost
 			time.Sleep(sleeptime)
 			continue
 		}
 
-		log.Info().Int("NumComment", currentCount).Uint64("SteamID", steamID).Msg("Found new comment")
+		if strings.Contains(currentCommentsPage.CommentsHTML, steamContentCheckText) {
+			log.Info().Uint64("ProfileID", steamID).Msg("Found new comment(s) still being checked by Steam, retrying in "+ sleeptime.String())
+			time.Sleep(sleeptime)
+			continue
+		}
 
-		player := getPlayerDetails(steamID)
+		log.Info().Uint64("ProfileID", steamID).Msg("Found new comment(s)")
+
+		profileOwner := getPlayerDetails(steamID)
+
+		for _, comment := range parseComments(currentCommentsPage) {
+			log.Debug().Interface("Comment", comment).Msg("Processing Comment")
+			if comment.Timestamp <= newestProcessedComment {
+				log.Debug().Uint64("CommentID", comment.ID).Msg("Skipping Comment")
+				continue
+			}
 
 			msg := tgbotapi.MessageConfig{
 				BaseChat:              tgbotapi.BaseChat{ChatID: config.BotConfig.ChatID},
-			ParseMode:             "html",
+				ParseMode:             "HTML",
-			DisableWebPagePreview: false,
+				DisableWebPagePreview: true,
-			Text:                  fmt.Sprintf(`New comment on <a href="%s">%s's</a> profile`, player.ProfileURL, player.PersonaName),
+				Text: fmt.Sprintf(`<b><a href="%s">%s</a> just commented on <a href="%s">%s</a>'s profile:</b>`, comment.AuthorProfileURL, comment.Author, profileOwner.ProfileURL, profileOwner.PersonaName) + "\n" +
+					"<blockquote>" + comment.Text + "</blockquote>",
+			}
+			log.Info().Interface("Comment", comment).Msg("Notifying about new Comment")
+			botIO.SendMessage(msg, bot)
+			time.Sleep(time.Minute / 20)
 		}
 
-		botIO.SendMessage(msg, bot)
+		newestProcessedComment = currentCommentsPage.TimeLastPost
-
-		previousCount = currentCount
 		time.Sleep(sleeptime)
 	}
 }

steam/types.go

@@ -1,9 +1,17 @@
 package steam
 
-type CommentResponse struct {
+type CommentsPage struct {
 	Success      bool   `json:"success"`
 	Start        int    `json:"start"`
 	TotalCount   int    `json:"total_count"`
 	CommentsHTML string `json:"comments_html"`
-	Timelastpost int    `json:"timelastpost"`
+	TimeLastPost int64  `json:"timelastpost"`
+}
+
+type Comment struct {
+	ID               uint64
+	Timestamp        int64
+	Author           string
+	AuthorProfileURL string
+	Text             string
 }