me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change all timestamps to use milliseconds instead of seconds

Browse Source
This commit is contained in:
Alessio 2024-01-07 15:15:43 -06:00
parent e300a882e4
commit 84ff6c12b7
10 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/webserver/handler_messages.go
View File

@ -61,7 +61,7 @@ func (app *Application) Messages(w http.ResponseWriter, r *http.Request) {
chat_view_data.MessageIDs = chat_contents.MessageIDs
if len(chat_view_data.MessageIDs) > 0 {
last_message_id := chat_view_data.MessageIDs[len(chat_view_data.MessageIDs)-1]
chat_view_data.LatestPollingTimestamp = int(chat_view_data.Messages[last_message_id].SentAt.Unix())
chat_view_data.LatestPollingTimestamp = int(chat_view_data.Messages[last_message_id].SentAt.UnixMilli())
}
if r.URL.Query().Has("poll") || len(parts) == 2 && parts[1] == "send" {

6
internal/webserver/server_test.go
View File

@ -113,7 +113,7 @@ func TestUserFeedWithCursor(t *testing.T) {
require := require.New(t)
// With a cursor
resp := do_request(httptest.NewRequest("GET", "/cernovich?cursor=1631935701", nil))
resp := do_request(httptest.NewRequest("GET", "/cernovich?cursor=1631935701000", nil))
require.Equal(resp.StatusCode, 200)
root, err := html.Parse(resp.Body)
@ -231,7 +231,7 @@ func TestTimelineWithCursor(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
resp := do_request(httptest.NewRequest("GET", "/timeline?cursor=1631935701", nil))
resp := do_request(httptest.NewRequest("GET", "/timeline?cursor=1631935701000", nil))
require.Equal(resp.StatusCode, 200)
root, err := html.Parse(resp.Body)
@ -293,7 +293,7 @@ func TestSearchWithCursor(t *testing.T) {
assert.Len(cascadia.QueryAll(root, selector(".timeline > .tweet")), 3)
// Add a cursor with the 1st tweet's posted_at time
resp = do_request(httptest.NewRequest("GET", "/search/who%20are?cursor=1628979529", nil))
resp = do_request(httptest.NewRequest("GET", "/search/who%20are?cursor=1628979529000", nil))
require.Equal(resp.StatusCode, 200)
root, err = html.Parse(resp.Body)
require.NoError(err)

6
pkg/persistence/compound_queries_test.go
View File

@ -48,7 +48,7 @@ func TestBuildUserFeed(t *testing.T) {
assert.Equal(feed.Items[1].TweetID, TweetID(1490116725395927042))
assert.Equal(feed.Items[1].RetweetID, TweetID(1490119308692766723))
assert.Equal(feed.CursorBottom.CursorValue, 1644107102)
assert.Equal(feed.CursorBottom.CursorValue, 1644107102000)
}
// Should load a feed in the middle (i.e., after some timestamp)
@ -62,7 +62,7 @@ func TestBuildUserFeedPage2(t *testing.T) {
c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
c.PageSize = 2
c.CursorPosition = persistence.CURSOR_MIDDLE
c.CursorValue = 1644107102
c.CursorValue = 1644107102000
feed, err := profile.NextPage(c, UserID(0))
require.NoError(err)
@ -88,7 +88,7 @@ func TestBuildUserFeedPage2(t *testing.T) {
assert.Equal(feed.Items[1].TweetID, TweetID(1453461248142495744))
assert.Equal(feed.Items[1].RetweetID, TweetID(0))
assert.Equal(feed.CursorBottom.CursorValue, 1635367140)
assert.Equal(feed.CursorBottom.CursorValue, 1635367140000)
}
// When the end of the feed is reached, an "End of feed" error should be raised

8
pkg/persistence/compound_ssf_queries_test.go
View File

@ -39,7 +39,7 @@ func TestCursorSearchByNewest(t *testing.T) {
assert.Equal(next_cursor.SortOrder, c.SortOrder)
assert.Equal(next_cursor.Keywords, c.Keywords)
assert.Equal(next_cursor.PageSize, c.PageSize)
assert.Equal(next_cursor.CursorValue, 1629520619)
assert.Equal(next_cursor.CursorValue, 1629520619000)
feed, err = profile.NextPage(next_cursor, UserID(0))
require.NoError(err)
@ -81,7 +81,7 @@ func TestCursorSearchWithRetweets(t *testing.T) {
assert.Equal(next_cursor.SortOrder, c.SortOrder)
assert.Equal(next_cursor.Keywords, c.Keywords)
assert.Equal(next_cursor.PageSize, c.PageSize)
assert.Equal(next_cursor.CursorValue, 1644111031)
assert.Equal(next_cursor.CursorValue, 1644111031000)
feed, err = profile.NextPage(next_cursor, UserID(0))
require.NoError(err)
@ -118,9 +118,9 @@ func TestTimeline(t *testing.T) {
assert.Equal(next_cursor.SortOrder, c.SortOrder)
assert.Equal(next_cursor.Keywords, c.Keywords)
assert.Equal(next_cursor.PageSize, c.PageSize)
assert.Equal(next_cursor.CursorValue, 1635367140)
assert.Equal(next_cursor.CursorValue, 1635367140000)
next_cursor.CursorValue = 1631935323 // Scroll down a bit, kind of randomly
next_cursor.CursorValue = 1631935323000 // Scroll down a bit, kind of randomly
feed, err = profile.NextPage(next_cursor, UserID(0))
require.NoError(err)

14
pkg/persistence/versions.go
View File

@ -212,6 +212,20 @@ var MIGRATIONS = []string{
);
create index if not exists index_follows_followee_id on follows (followee_id);
create index if not exists index_follows_follower_id on follows (follower_id);`,
`update tweets set
posted_at = posted_at * 1000,
last_scraped_at = last_scraped_at * 1000;
update users set join_date = join_date * 1000;
update spaces set
created_at = created_at * 1000,
started_at = started_at * 1000,
ended_at = ended_at * 1000,
updated_at = updated_at * 1000;
update retweets set retweeted_at = retweeted_at * 1000;
update polls set
voting_ends_at = voting_ends_at * 1000,
last_scraped_at = last_scraped_at * 1000;
update chat_rooms set created_at = created_at * 1000;`,
}
var ENGINE_DATABASE_VERSION = len(MIGRATIONS)

10
pkg/scraper/api_types_dms_test.go
View File

@ -23,7 +23,7 @@ func TestParseAPIDMMessage(t *testing.T) {
message := ParseAPIDMMessage(api_message)
assert.Equal(message.ID, DMMessageID(1663623203644751885))
assert.Equal(message.SentAt, TimestampFromUnix(1685473655064))
assert.Equal(message.SentAt, TimestampFromUnixMilli(1685473655064))
assert.Equal(message.DMChatRoomID, DMChatRoomID("1458284524761075714-1488963321701171204"))
assert.Equal(message.SenderID, UserID(1458284524761075714))
assert.Equal(message.Text, "Yeah i know who you are lol")
@ -48,7 +48,7 @@ func TestParseAPIDMMessageWithReaction(t *testing.T) {
reacc, is_ok := message.Reactions[UserID(1458284524761075714)]
require.True(t, is_ok)
assert.Equal(reacc.ID, DMMessageID(1665914315742781440))
assert.Equal(reacc.SentAt, TimestampFromUnix(1686019898732))
assert.Equal(reacc.SentAt, TimestampFromUnixMilli(1686019898732))
assert.Equal(reacc.DMMessageID, DMMessageID(1663623062195957773))
assert.Equal(reacc.SenderID, UserID(1458284524761075714))
assert.Equal(reacc.Emoji, "😂")
@ -96,7 +96,7 @@ func TestParseAPIDMConversation(t *testing.T) {
chat_room := ParseAPIDMChatRoom(api_room)
assert.Equal(DMChatRoomID("1458284524761075714-1488963321701171204"), chat_room.ID)
assert.Equal("ONE_TO_ONE", chat_room.Type)
assert.Equal(TimestampFromUnix(1686025129086), chat_room.LastMessagedAt)
assert.Equal(TimestampFromUnixMilli(1686025129086), chat_room.LastMessagedAt)
assert.False(chat_room.IsNSFW)
assert.Len(chat_room.Participants, 2)
@ -132,11 +132,11 @@ func TestParseAPIDMGroupChat(t *testing.T) {
chat_room := ParseAPIDMChatRoom(api_room)
assert.Equal(DMChatRoomID("1710215025518948715"), chat_room.ID)
assert.Equal("GROUP_DM", chat_room.Type)
assert.Equal(TimestampFromUnix(1700112789457), chat_room.LastMessagedAt)
assert.Equal(TimestampFromUnixMilli(1700112789457), chat_room.LastMessagedAt)
assert.False(chat_room.IsNSFW)
// Group DM settings
assert.Equal(chat_room.CreatedAt, TimestampFromUnix(1696582011))
assert.Equal(chat_room.CreatedAt, TimestampFromUnixMilli(1696582011037))
assert.Equal(chat_room.CreatedByUserID, UserID(2694459866))
assert.Equal(chat_room.Name, "Schön ist die Welt")
assert.Equal(chat_room.AvatarImageRemoteURL,

8
pkg/scraper/api_types_spaces.go
View File

@ -64,10 +64,10 @@ func (r SpaceResponse) ToTweetTrove() TweetTrove {
space.Title = data.Metadata.Title
space.State = data.Metadata.State
space.CreatedById = UserID(data.Metadata.CreatorResults.Result.ID)
space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt / 1000)
space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt / 1000)
space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt / 1000)
space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt / 1000)
space.CreatedAt = TimestampFromUnixMilli(data.Metadata.CreatedAt)
space.StartedAt = TimestampFromUnixMilli(data.Metadata.StartedAt)
space.EndedAt = TimestampFromUnixMilli(data.Metadata.EndedAt)
space.UpdatedAt = TimestampFromUnixMilli(data.Metadata.UpdatedAt)
space.IsAvailableForReplay = data.Metadata.IsSpaceAvailableForReplay
space.ReplayWatchCount = data.Metadata.TotalReplayWatched
space.LiveListenersCount = data.Metadata.TotalLiveListeners

4
pkg/scraper/dm_chat_room.go
View File

@ -49,11 +49,11 @@ func ParseAPIDMChatRoom(api_room APIDMConversation) DMChatRoom {
ret := DMChatRoom{}
ret.ID = DMChatRoomID(api_room.ConversationID)
ret.Type = api_room.Type
ret.LastMessagedAt = TimestampFromUnix(int64(api_room.SortTimestamp))
ret.LastMessagedAt = TimestampFromUnixMilli(int64(api_room.SortTimestamp))
ret.IsNSFW = api_room.NSFW
if ret.Type == "GROUP_DM" {
ret.CreatedAt = TimestampFromUnix(int64(api_room.CreateTime / 1000))
ret.CreatedAt = TimestampFromUnixMilli(int64(api_room.CreateTime))
ret.CreatedByUserID = UserID(api_room.CreatedByUserID)
ret.Name = api_room.Name
ret.AvatarImageRemoteURL = api_room.AvatarImage

4
pkg/scraper/dm_message.go
View File

@ -14,7 +14,7 @@ func ParseAPIDMReaction(reacc APIDMReaction) DMReaction {
ret := DMReaction{}
ret.ID = DMMessageID(reacc.ID)
ret.SenderID = UserID(reacc.SenderID)
ret.SentAt = TimestampFromUnix(int64(reacc.Time))
ret.SentAt = TimestampFromUnixMilli(int64(reacc.Time))
ret.Emoji = reacc.Emoji
return ret
}
@ -34,7 +34,7 @@ type DMMessage struct {
func ParseAPIDMMessage(message APIDMMessage) DMMessage {
ret := DMMessage{}
ret.ID = DMMessageID(message.ID)
ret.SentAt = TimestampFromUnix(int64(message.Time))
ret.SentAt = TimestampFromUnixMilli(int64(message.Time))
ret.DMChatRoomID = DMChatRoomID(message.ConversationID)
ret.SenderID = UserID(message.MessageData.SenderID)
ret.Text = message.MessageData.Text

7
pkg/scraper/timestamp_type.go
View File

@ -11,7 +11,7 @@ type Timestamp struct {
}
func (t Timestamp) Value() (driver.Value, error) {
return t.Unix(), nil
return t.UnixMilli(), nil
}
func (t *Timestamp) Scan(src interface{}) error {
@ -19,7 +19,7 @@ func (t *Timestamp) Scan(src interface{}) error {
if !is_ok {
return fmt.Errorf("Incompatible type for Timestamp: %#v", src)
}
*t = Timestamp{time.Unix(val, 0)}
*t = Timestamp{time.UnixMilli(val)}
return nil
}
@ -38,3 +38,6 @@ func TimestampFromString(s string) (Timestamp, error) {
func TimestampFromUnix(num int64) Timestamp {
return Timestamp{time.Unix(num, 0)}
}
func TimestampFromUnixMilli(num int64) Timestamp {
return Timestamp{time.UnixMilli(num)}
}