-
-
Notifications
You must be signed in to change notification settings - Fork 49
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Spam storage and web ui for detected spam (#51)
* Add spam logging to database The main functionality of the spam logger has been expanded to not only log spam messages to a file, but also store them in a database. This facilitates efficient tracking and analysis of spam data. Furthermore, test cases have been added to ensure this new functionality works as expected. * Add Detected Spam page to web UI A Detected Spam page has been added to the web UI, where users can see a list of detected spam messages. Changes have been made to attach data DB to the server activation function and the timestamp is now localized. Fixed a bug in storage/detected_spam.go where the 'Checks' field was incorrectly attached to a non-existent 'entry' variable. * Add DetectedSpamReader mock and implement detected spam HTML handler tests
- Loading branch information
Showing
11 changed files
with
512 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package storage | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "log" | ||
| "time" | ||
|
|
||
| "github.com/jmoiron/sqlx" | ||
|
|
||
| "github.com/umputun/tg-spam/lib/spamcheck" | ||
| ) | ||
|
|
||
| // DetectedSpam is a storage for detected spam entries | ||
| type DetectedSpam struct { | ||
| db *sqlx.DB | ||
| } | ||
|
|
||
| // DetectedSpamInfo represents information about a detected spam entry. | ||
| type DetectedSpamInfo struct { | ||
| Text string `db:"text"` | ||
| UserID int64 `db:"user_id"` | ||
| UserName string `db:"user_name"` | ||
| Timestamp time.Time `db:"timestamp"` | ||
| ChecksJSON string `db:"checks"` // Store as JSON | ||
| Checks []spamcheck.Response `db:"-"` // Don't store in DB | ||
| } | ||
|
|
||
| // NewDetectedSpam creates a new DetectedSpam storage | ||
| func NewDetectedSpam(db *sqlx.DB) (*DetectedSpam, error) { | ||
| _, err := db.Exec(`CREATE TABLE IF NOT EXISTS detected_spam ( | ||
| id INTEGER PRIMARY KEY AUTOINCREMENT, | ||
| text TEXT, | ||
| user_id INTEGER, | ||
| user_name TEXT, | ||
| timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, | ||
| checks TEXT | ||
| )`) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create detected_spam table: %w", err) | ||
| } | ||
| return &DetectedSpam{db: db}, nil | ||
| } | ||
|
|
||
| // Write adds a new detected spam entry | ||
| func (ds *DetectedSpam) Write(entry DetectedSpamInfo, checks []spamcheck.Response) error { | ||
| checksJSON, err := json.Marshal(checks) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to marshal checks: %w", err) | ||
| } | ||
|
|
||
| query := `INSERT INTO detected_spam (text, user_id, user_name, timestamp, checks) VALUES (?, ?, ?, ?, ?)` | ||
| if _, err := ds.db.Exec(query, entry.Text, entry.UserID, entry.UserName, entry.Timestamp, checksJSON); err != nil { | ||
| return fmt.Errorf("failed to insert detected spam entry: %w", err) | ||
| } | ||
|
|
||
| log.Printf("[INFO] detected spam entry added for user_id:%d, name:%s", entry.UserID, entry.UserName) | ||
| return nil | ||
| } | ||
|
|
||
| // Read returns all detected spam entries | ||
| func (ds *DetectedSpam) Read() ([]DetectedSpamInfo, error) { | ||
| var entries []DetectedSpamInfo | ||
| err := ds.db.Select(&entries, "SELECT text, user_id, user_name, timestamp, checks FROM detected_spam ORDER BY timestamp DESC") | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get detected spam entries: %w", err) | ||
| } | ||
|
|
||
| for i, entry := range entries { | ||
| var checks []spamcheck.Response | ||
| if err := json.Unmarshal([]byte(entry.ChecksJSON), &checks); err != nil { | ||
| return nil, fmt.Errorf("failed to unmarshal checks for entry %d: %w", i, err) | ||
| } | ||
| entries[i].Checks = checks | ||
| entries[i].Timestamp = entry.Timestamp.Local() | ||
| } | ||
| return entries, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| package storage | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/jmoiron/sqlx" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/umputun/tg-spam/lib/spamcheck" | ||
| ) | ||
|
|
||
| func TestDetectedSpam_NewDetectedSpam(t *testing.T) { | ||
| db, err := sqlx.Open("sqlite", ":memory:") | ||
| require.NoError(t, err) | ||
| defer db.Close() | ||
|
|
||
| _, err = NewDetectedSpam(db) | ||
| require.NoError(t, err) | ||
|
|
||
| var exists int | ||
| err = db.Get(&exists, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='detected_spam'") | ||
| require.NoError(t, err) | ||
| assert.Equal(t, 1, exists) | ||
| } | ||
|
|
||
| func TestDetectedSpam_Write(t *testing.T) { | ||
| db, err := sqlx.Open("sqlite", ":memory:") | ||
| require.NoError(t, err) | ||
| defer db.Close() | ||
|
|
||
| ds, err := NewDetectedSpam(db) | ||
| require.NoError(t, err) | ||
|
|
||
| spamEntry := DetectedSpamInfo{ | ||
| Text: "spam message", | ||
| UserID: 1, | ||
| UserName: "Spammer", | ||
| Timestamp: time.Now(), | ||
| } | ||
|
|
||
| checks := []spamcheck.Response{ | ||
| { | ||
| Name: "Check1", | ||
| Spam: true, | ||
| Details: "Details 1", | ||
| }, | ||
| } | ||
|
|
||
| err = ds.Write(spamEntry, checks) | ||
| require.NoError(t, err) | ||
|
|
||
| var count int | ||
| err = db.Get(&count, "SELECT COUNT(*) FROM detected_spam") | ||
| require.NoError(t, err) | ||
| assert.Equal(t, 1, count) | ||
| } | ||
|
|
||
| func TestDetectedSpam_Read(t *testing.T) { | ||
| db, err := sqlx.Open("sqlite", ":memory:") | ||
| require.NoError(t, err) | ||
| defer db.Close() | ||
|
|
||
| ds, err := NewDetectedSpam(db) | ||
| require.NoError(t, err) | ||
|
|
||
| spamEntry := DetectedSpamInfo{ | ||
| Text: "spam message", | ||
| UserID: 1, | ||
| UserName: "Spammer", | ||
| Timestamp: time.Now(), | ||
| } | ||
|
|
||
| checks := []spamcheck.Response{ | ||
| { | ||
| Name: "Check1", | ||
| Spam: true, | ||
| Details: "Details 1", | ||
| }, | ||
| } | ||
|
|
||
| checksJSON, err := json.Marshal(checks) | ||
| require.NoError(t, err) | ||
| _, err = db.Exec("INSERT INTO detected_spam (text, user_id, user_name, timestamp, checks) VALUES (?, ?, ?, ?, ?)", spamEntry.Text, spamEntry.UserID, spamEntry.UserName, spamEntry.Timestamp, checksJSON) | ||
| require.NoError(t, err) | ||
|
|
||
| entries, err := ds.Read() | ||
| require.NoError(t, err) | ||
| require.Len(t, entries, 1) | ||
|
|
||
| assert.Equal(t, spamEntry.Text, entries[0].Text) | ||
| assert.Equal(t, spamEntry.UserID, entries[0].UserID) | ||
| assert.Equal(t, spamEntry.UserName, entries[0].UserName) | ||
|
|
||
| var retrievedChecks []spamcheck.Response | ||
| err = json.Unmarshal([]byte(entries[0].ChecksJSON), &retrievedChecks) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, checks, retrievedChecks) | ||
| t.Logf("retrieved checks: %+v", retrievedChecks) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.