From 9c64a3e4b55150ee953d5835769a67aa45aa927e Mon Sep 17 00:00:00 2001 From: Noah Theus Date: Sat, 21 Dec 2024 08:06:47 +0100 Subject: [PATCH] v0.2 - Include comment text in notification - Include commenter in notification - Use Timestamp to check if a comment is new - Set sleep interval between requests in env variable --- config/types.go | 1 + docker-compose.yaml | 3 +- go.mod | 6 ++-- go.sum | 67 +++++++++++++++++++++++++++++++++++++++++++++ steam/api.go | 2 +- steam/http.go | 50 ++++++++++++++++++++++++++++++--- steam/profile.go | 59 +++++++++++++++++++++++++++------------ steam/types.go | 12 ++++++-- 8 files changed, 172 insertions(+), 28 deletions(-) diff --git a/config/types.go b/config/types.go index a80658c..5abf464 100644 --- a/config/types.go +++ b/config/types.go @@ -6,4 +6,5 @@ type config struct { DebugMode bool `env:"DEBUGMODE, default=false"` ChatID int64 `env:"CHATID"` Watchers []uint64 `env:"WATCHERS"` + SleepInterval int `env:"SLEEPINTERVAL"` } diff --git a/docker-compose.yaml b/docker-compose.yaml index 4209b24..e3906bc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,4 +10,5 @@ services: - STEAMAPIKEY= - DebugMode=false - CHATID=123 - - WATCHERS=123,456,789 \ No newline at end of file + - WATCHERS=123,456,789 + - SLEEPINTERVAL=60 \ No newline at end of file diff --git a/go.mod b/go.mod index 66ba8b4..e6693ba 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,17 @@ module watn3y/steamsalty go 1.23.4 require ( + github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1 + github.com/PuerkitoBio/goquery v1.10.0 github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 github.com/rs/zerolog v1.33.0 github.com/sethvargo/go-envconfig v1.1.0 - github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1 ) require ( - + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect ) diff --git a/go.sum b/go.sum index 1a1a2d7..696f506 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1 h1:PD13eMe9XAgPQ0SYWyirqwyOJG90TlEWApCw8A699l0= github.com/Philipp15b/go-steamapi v0.0.0-20210114153316-ec4fdd23b4c1/go.mod h1:eQR7Xf64m2ALDAQE7Nr9ylFZhav1izvF3zzysKPhb0I= +github.com/PuerkitoBio/goquery v1.10.0 h1:6fiXdLuUvYs2OJSvNRqlNPoBm6YABE226xrbavY5Wv4= +github.com/PuerkitoBio/goquery v1.10.0/go.mod h1:TjZZl68Q3eGHNBA8CWaxAN7rOU1EbDz3CWuolcO5Yu4= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc= github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8= @@ -18,8 +22,71 @@ github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog= github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/steam/api.go b/steam/api.go index db12e11..80bc253 100644 --- a/steam/api.go +++ b/steam/api.go @@ -12,6 +12,6 @@ func getPlayerDetails(steamID uint64) (summary steamapi.PlayerSummary) { if err != nil { log.Error().Err(err).Msg("Failed to get Player Summary") } - + log.Debug().Interface("Player", response[0]).Msg("Successfully got PlayerSummary from Steam API") return response[0] } diff --git a/steam/http.go b/steam/http.go index 28e883d..70b0337 100644 --- a/steam/http.go +++ b/steam/http.go @@ -2,14 +2,18 @@ package steam import ( "encoding/json" - "github.com/rs/zerolog/log" "io" "net/http" "net/url" + "slices" "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/rs/zerolog/log" ) -func getComments(steamID uint64, start int, count int) (comments CommentResponse) { +func getComments(steamID uint64, start int, count int) (page CommentsPage) { baseURL := "https://steamcommunity.com/comment/Profile/render/" @@ -38,11 +42,49 @@ func getComments(steamID uint64, start int, count int) (comments CommentResponse log.Trace().Interface("Body", resp.Body) } - err = json.Unmarshal(body, &comments) + err = json.Unmarshal(body, &page) if err != nil { log.Error().Err(err).Msg("Failed to parse Comments as JSON") } - log.Debug().Interface("CommentPage", comments).Msg("Successfully got Comment Page") + log.Debug().Interface("CommentPage", page).Uint64("ProfileID", steamID).Msg("Successfully got Comment Page") + + return page +} + +func parseComments(rawComments CommentsPage) (comments []Comment) { + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(rawComments.CommentsHTML)) + if err != nil { + log.Error().Err(err).Msg("Error while parsing CommentsHTML") + return + } + doc.Find(".commentthread_comment.responsive_body_text").Each(func(i int, s *goquery.Selection) { + var c Comment + + parsedID, err := strconv.ParseUint(strings.TrimPrefix(s.AttrOr("id", ""), "comment_"), 10, 64) + if err != nil { + log.Error().Err(err).Msg("Error while parsing Comment ID") + return + } + c.ID = parsedID + + c.Timestamp, err = strconv.ParseInt(s.Find(".commentthread_comment_timestamp").AttrOr("data-timestamp", ""), 10, 64) + if err != nil { + log.Error().Err(err).Msg("Error while parsing Comment Timestamp") + return + } + + c.Author = s.Find(".commentthread_comment_author .hoverunderline bdi").Text() + + c.AuthorProfileURL, _ = s.Find(".commentthread_comment_author .hoverunderline").Attr("href") + + c.Text = strings.TrimSpace(s.Find(".commentthread_comment_text").Text()) + + comments = append(comments, c) + }) + + slices.Reverse(comments) + log.Debug().Interface("Comments", comments).Msg("Successfully parsed Comment Page") return comments } diff --git a/steam/profile.go b/steam/profile.go index 648ed60..24ee41b 100644 --- a/steam/profile.go +++ b/steam/profile.go @@ -2,16 +2,21 @@ package steam import ( "fmt" - tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5" - "github.com/rs/zerolog/log" "math" + + "strings" "sync" "time" + "watn3y/steamsalty/botIO" "watn3y/steamsalty/config" + + tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5" + "github.com/rs/zerolog/log" ) -var sleeptime time.Duration = 1 * time.Minute +var sleeptime time.Duration = time.Duration(config.BotConfig.SleepInterval) * time.Second +var steamContentCheckText string = "This comment is awaiting analysis by our automated content check system. It will be temporarily hidden until we verify that it does not contain harmful content (e.g. links to websites that attempt to steal information)." func StartWatchers(bot *tgbotapi.BotAPI) { var wg sync.WaitGroup @@ -29,29 +34,47 @@ func StartWatchers(bot *tgbotapi.BotAPI) { func watcher(bot *tgbotapi.BotAPI, steamID uint64) { log.Info().Uint64("SteamID", steamID).Msg("Started Watcher") - var previousCount int + + var newestProcessedComment int64 = 0 + for { - currentCount := getComments(steamID, math.MaxInt32, 0).TotalCount - if previousCount == 0 || currentCount <= previousCount { - previousCount = currentCount + currentCommentsPage := getComments(steamID, 0, math.MaxInt32) + if newestProcessedComment == 0 || newestProcessedComment == currentCommentsPage.TimeLastPost { + newestProcessedComment = currentCommentsPage.TimeLastPost time.Sleep(sleeptime) continue } - log.Info().Int("NumComment", currentCount).Uint64("SteamID", steamID).Msg("Found new comment") - - player := getPlayerDetails(steamID) - - msg := tgbotapi.MessageConfig{ - BaseChat: tgbotapi.BaseChat{ChatID: config.BotConfig.ChatID}, - ParseMode: "html", - DisableWebPagePreview: false, - Text: fmt.Sprintf(`New comment on %s's profile`, player.ProfileURL, player.PersonaName), + if strings.Contains(currentCommentsPage.CommentsHTML, steamContentCheckText) { + log.Info().Uint64("ProfileID", steamID).Msg("Found new comment(s) still being checked by Steam, retrying in "+ sleeptime.String()) + time.Sleep(sleeptime) + continue } - botIO.SendMessage(msg, bot) + log.Info().Uint64("ProfileID", steamID).Msg("Found new comment(s)") - previousCount = currentCount + profileOwner := getPlayerDetails(steamID) + + for _, comment := range parseComments(currentCommentsPage) { + log.Debug().Interface("Comment", comment).Msg("Processing Comment") + if comment.Timestamp <= newestProcessedComment { + log.Debug().Uint64("CommentID", comment.ID).Msg("Skipping Comment") + continue + } + + msg := tgbotapi.MessageConfig{ + BaseChat: tgbotapi.BaseChat{ChatID: config.BotConfig.ChatID}, + ParseMode: "HTML", + DisableWebPagePreview: true, + Text: fmt.Sprintf(`%s just commented on %s's profile:`, comment.AuthorProfileURL, comment.Author, profileOwner.ProfileURL, profileOwner.PersonaName) + "\n" + + "
" + comment.Text + "
", + } + log.Info().Interface("Comment", comment).Msg("Notifying about new Comment") + botIO.SendMessage(msg, bot) + time.Sleep(time.Minute / 20) + } + + newestProcessedComment = currentCommentsPage.TimeLastPost time.Sleep(sleeptime) } } diff --git a/steam/types.go b/steam/types.go index 4d3ae86..1445262 100644 --- a/steam/types.go +++ b/steam/types.go @@ -1,9 +1,17 @@ package steam -type CommentResponse struct { +type CommentsPage struct { Success bool `json:"success"` Start int `json:"start"` TotalCount int `json:"total_count"` CommentsHTML string `json:"comments_html"` - Timelastpost int `json:"timelastpost"` + TimeLastPost int64 `json:"timelastpost"` +} + +type Comment struct { + ID uint64 + Timestamp int64 + Author string + AuthorProfileURL string + Text string }