diff --git a/internal/webserver/handler_messages.go b/internal/webserver/handler_messages.go index 929aee6..b5a97a1 100644 --- a/internal/webserver/handler_messages.go +++ b/internal/webserver/handler_messages.go @@ -61,7 +61,7 @@ func (app *Application) Messages(w http.ResponseWriter, r *http.Request) { chat_view_data.MessageIDs = chat_contents.MessageIDs if len(chat_view_data.MessageIDs) > 0 { last_message_id := chat_view_data.MessageIDs[len(chat_view_data.MessageIDs)-1] - chat_view_data.LatestPollingTimestamp = int(chat_view_data.Messages[last_message_id].SentAt.Unix()) + chat_view_data.LatestPollingTimestamp = int(chat_view_data.Messages[last_message_id].SentAt.UnixMilli()) } if r.URL.Query().Has("poll") || len(parts) == 2 && parts[1] == "send" { diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go index 1912a41..ed6a427 100644 --- a/internal/webserver/server_test.go +++ b/internal/webserver/server_test.go @@ -113,7 +113,7 @@ func TestUserFeedWithCursor(t *testing.T) { require := require.New(t) // With a cursor - resp := do_request(httptest.NewRequest("GET", "/cernovich?cursor=1631935701", nil)) + resp := do_request(httptest.NewRequest("GET", "/cernovich?cursor=1631935701000", nil)) require.Equal(resp.StatusCode, 200) root, err := html.Parse(resp.Body) @@ -231,7 +231,7 @@ func TestTimelineWithCursor(t *testing.T) { assert := assert.New(t) require := require.New(t) - resp := do_request(httptest.NewRequest("GET", "/timeline?cursor=1631935701", nil)) + resp := do_request(httptest.NewRequest("GET", "/timeline?cursor=1631935701000", nil)) require.Equal(resp.StatusCode, 200) root, err := html.Parse(resp.Body) @@ -293,7 +293,7 @@ func TestSearchWithCursor(t *testing.T) { assert.Len(cascadia.QueryAll(root, selector(".timeline > .tweet")), 3) // Add a cursor with the 1st tweet's posted_at time - resp = do_request(httptest.NewRequest("GET", "/search/who%20are?cursor=1628979529", nil)) + resp = do_request(httptest.NewRequest("GET", "/search/who%20are?cursor=1628979529000", nil)) require.Equal(resp.StatusCode, 200) root, err = html.Parse(resp.Body) require.NoError(err) diff --git a/pkg/persistence/compound_queries_test.go b/pkg/persistence/compound_queries_test.go index d1f6063..89d9b50 100644 --- a/pkg/persistence/compound_queries_test.go +++ b/pkg/persistence/compound_queries_test.go @@ -48,7 +48,7 @@ func TestBuildUserFeed(t *testing.T) { assert.Equal(feed.Items[1].TweetID, TweetID(1490116725395927042)) assert.Equal(feed.Items[1].RetweetID, TweetID(1490119308692766723)) - assert.Equal(feed.CursorBottom.CursorValue, 1644107102) + assert.Equal(feed.CursorBottom.CursorValue, 1644107102000) } // Should load a feed in the middle (i.e., after some timestamp) @@ -62,7 +62,7 @@ func TestBuildUserFeedPage2(t *testing.T) { c := persistence.NewUserFeedCursor(UserHandle("cernovich")) c.PageSize = 2 c.CursorPosition = persistence.CURSOR_MIDDLE - c.CursorValue = 1644107102 + c.CursorValue = 1644107102000 feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) @@ -88,7 +88,7 @@ func TestBuildUserFeedPage2(t *testing.T) { assert.Equal(feed.Items[1].TweetID, TweetID(1453461248142495744)) assert.Equal(feed.Items[1].RetweetID, TweetID(0)) - assert.Equal(feed.CursorBottom.CursorValue, 1635367140) + assert.Equal(feed.CursorBottom.CursorValue, 1635367140000) } // When the end of the feed is reached, an "End of feed" error should be raised diff --git a/pkg/persistence/compound_ssf_queries_test.go b/pkg/persistence/compound_ssf_queries_test.go index f9e88e5..020d51e 100644 --- a/pkg/persistence/compound_ssf_queries_test.go +++ b/pkg/persistence/compound_ssf_queries_test.go @@ -39,7 +39,7 @@ func TestCursorSearchByNewest(t *testing.T) { assert.Equal(next_cursor.SortOrder, c.SortOrder) assert.Equal(next_cursor.Keywords, c.Keywords) assert.Equal(next_cursor.PageSize, c.PageSize) - assert.Equal(next_cursor.CursorValue, 1629520619) + assert.Equal(next_cursor.CursorValue, 1629520619000) feed, err = profile.NextPage(next_cursor, UserID(0)) require.NoError(err) @@ -81,7 +81,7 @@ func TestCursorSearchWithRetweets(t *testing.T) { assert.Equal(next_cursor.SortOrder, c.SortOrder) assert.Equal(next_cursor.Keywords, c.Keywords) assert.Equal(next_cursor.PageSize, c.PageSize) - assert.Equal(next_cursor.CursorValue, 1644111031) + assert.Equal(next_cursor.CursorValue, 1644111031000) feed, err = profile.NextPage(next_cursor, UserID(0)) require.NoError(err) @@ -118,9 +118,9 @@ func TestTimeline(t *testing.T) { assert.Equal(next_cursor.SortOrder, c.SortOrder) assert.Equal(next_cursor.Keywords, c.Keywords) assert.Equal(next_cursor.PageSize, c.PageSize) - assert.Equal(next_cursor.CursorValue, 1635367140) + assert.Equal(next_cursor.CursorValue, 1635367140000) - next_cursor.CursorValue = 1631935323 // Scroll down a bit, kind of randomly + next_cursor.CursorValue = 1631935323000 // Scroll down a bit, kind of randomly feed, err = profile.NextPage(next_cursor, UserID(0)) require.NoError(err) diff --git a/pkg/persistence/versions.go b/pkg/persistence/versions.go index e2d64db..6790c77 100644 --- a/pkg/persistence/versions.go +++ b/pkg/persistence/versions.go @@ -212,6 +212,20 @@ var MIGRATIONS = []string{ ); create index if not exists index_follows_followee_id on follows (followee_id); create index if not exists index_follows_follower_id on follows (follower_id);`, + `update tweets set + posted_at = posted_at * 1000, + last_scraped_at = last_scraped_at * 1000; + update users set join_date = join_date * 1000; + update spaces set + created_at = created_at * 1000, + started_at = started_at * 1000, + ended_at = ended_at * 1000, + updated_at = updated_at * 1000; + update retweets set retweeted_at = retweeted_at * 1000; + update polls set + voting_ends_at = voting_ends_at * 1000, + last_scraped_at = last_scraped_at * 1000; + update chat_rooms set created_at = created_at * 1000;`, } var ENGINE_DATABASE_VERSION = len(MIGRATIONS) diff --git a/pkg/scraper/api_types_dms_test.go b/pkg/scraper/api_types_dms_test.go index 8238de3..3f7f1b9 100644 --- a/pkg/scraper/api_types_dms_test.go +++ b/pkg/scraper/api_types_dms_test.go @@ -23,7 +23,7 @@ func TestParseAPIDMMessage(t *testing.T) { message := ParseAPIDMMessage(api_message) assert.Equal(message.ID, DMMessageID(1663623203644751885)) - assert.Equal(message.SentAt, TimestampFromUnix(1685473655064)) + assert.Equal(message.SentAt, TimestampFromUnixMilli(1685473655064)) assert.Equal(message.DMChatRoomID, DMChatRoomID("1458284524761075714-1488963321701171204")) assert.Equal(message.SenderID, UserID(1458284524761075714)) assert.Equal(message.Text, "Yeah i know who you are lol") @@ -48,7 +48,7 @@ func TestParseAPIDMMessageWithReaction(t *testing.T) { reacc, is_ok := message.Reactions[UserID(1458284524761075714)] require.True(t, is_ok) assert.Equal(reacc.ID, DMMessageID(1665914315742781440)) - assert.Equal(reacc.SentAt, TimestampFromUnix(1686019898732)) + assert.Equal(reacc.SentAt, TimestampFromUnixMilli(1686019898732)) assert.Equal(reacc.DMMessageID, DMMessageID(1663623062195957773)) assert.Equal(reacc.SenderID, UserID(1458284524761075714)) assert.Equal(reacc.Emoji, "😂") @@ -96,7 +96,7 @@ func TestParseAPIDMConversation(t *testing.T) { chat_room := ParseAPIDMChatRoom(api_room) assert.Equal(DMChatRoomID("1458284524761075714-1488963321701171204"), chat_room.ID) assert.Equal("ONE_TO_ONE", chat_room.Type) - assert.Equal(TimestampFromUnix(1686025129086), chat_room.LastMessagedAt) + assert.Equal(TimestampFromUnixMilli(1686025129086), chat_room.LastMessagedAt) assert.False(chat_room.IsNSFW) assert.Len(chat_room.Participants, 2) @@ -132,11 +132,11 @@ func TestParseAPIDMGroupChat(t *testing.T) { chat_room := ParseAPIDMChatRoom(api_room) assert.Equal(DMChatRoomID("1710215025518948715"), chat_room.ID) assert.Equal("GROUP_DM", chat_room.Type) - assert.Equal(TimestampFromUnix(1700112789457), chat_room.LastMessagedAt) + assert.Equal(TimestampFromUnixMilli(1700112789457), chat_room.LastMessagedAt) assert.False(chat_room.IsNSFW) // Group DM settings - assert.Equal(chat_room.CreatedAt, TimestampFromUnix(1696582011)) + assert.Equal(chat_room.CreatedAt, TimestampFromUnixMilli(1696582011037)) assert.Equal(chat_room.CreatedByUserID, UserID(2694459866)) assert.Equal(chat_room.Name, "Schön ist die Welt") assert.Equal(chat_room.AvatarImageRemoteURL, diff --git a/pkg/scraper/api_types_spaces.go b/pkg/scraper/api_types_spaces.go index aad0292..8b6c4fc 100644 --- a/pkg/scraper/api_types_spaces.go +++ b/pkg/scraper/api_types_spaces.go @@ -64,10 +64,10 @@ func (r SpaceResponse) ToTweetTrove() TweetTrove { space.Title = data.Metadata.Title space.State = data.Metadata.State space.CreatedById = UserID(data.Metadata.CreatorResults.Result.ID) - space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt / 1000) - space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt / 1000) - space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt / 1000) - space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt / 1000) + space.CreatedAt = TimestampFromUnixMilli(data.Metadata.CreatedAt) + space.StartedAt = TimestampFromUnixMilli(data.Metadata.StartedAt) + space.EndedAt = TimestampFromUnixMilli(data.Metadata.EndedAt) + space.UpdatedAt = TimestampFromUnixMilli(data.Metadata.UpdatedAt) space.IsAvailableForReplay = data.Metadata.IsSpaceAvailableForReplay space.ReplayWatchCount = data.Metadata.TotalReplayWatched space.LiveListenersCount = data.Metadata.TotalLiveListeners diff --git a/pkg/scraper/dm_chat_room.go b/pkg/scraper/dm_chat_room.go index a98caec..5f26faa 100644 --- a/pkg/scraper/dm_chat_room.go +++ b/pkg/scraper/dm_chat_room.go @@ -49,11 +49,11 @@ func ParseAPIDMChatRoom(api_room APIDMConversation) DMChatRoom { ret := DMChatRoom{} ret.ID = DMChatRoomID(api_room.ConversationID) ret.Type = api_room.Type - ret.LastMessagedAt = TimestampFromUnix(int64(api_room.SortTimestamp)) + ret.LastMessagedAt = TimestampFromUnixMilli(int64(api_room.SortTimestamp)) ret.IsNSFW = api_room.NSFW if ret.Type == "GROUP_DM" { - ret.CreatedAt = TimestampFromUnix(int64(api_room.CreateTime / 1000)) + ret.CreatedAt = TimestampFromUnixMilli(int64(api_room.CreateTime)) ret.CreatedByUserID = UserID(api_room.CreatedByUserID) ret.Name = api_room.Name ret.AvatarImageRemoteURL = api_room.AvatarImage diff --git a/pkg/scraper/dm_message.go b/pkg/scraper/dm_message.go index fedbc68..450f979 100644 --- a/pkg/scraper/dm_message.go +++ b/pkg/scraper/dm_message.go @@ -14,7 +14,7 @@ func ParseAPIDMReaction(reacc APIDMReaction) DMReaction { ret := DMReaction{} ret.ID = DMMessageID(reacc.ID) ret.SenderID = UserID(reacc.SenderID) - ret.SentAt = TimestampFromUnix(int64(reacc.Time)) + ret.SentAt = TimestampFromUnixMilli(int64(reacc.Time)) ret.Emoji = reacc.Emoji return ret } @@ -34,7 +34,7 @@ type DMMessage struct { func ParseAPIDMMessage(message APIDMMessage) DMMessage { ret := DMMessage{} ret.ID = DMMessageID(message.ID) - ret.SentAt = TimestampFromUnix(int64(message.Time)) + ret.SentAt = TimestampFromUnixMilli(int64(message.Time)) ret.DMChatRoomID = DMChatRoomID(message.ConversationID) ret.SenderID = UserID(message.MessageData.SenderID) ret.Text = message.MessageData.Text diff --git a/pkg/scraper/timestamp_type.go b/pkg/scraper/timestamp_type.go index 0aa2ec1..6e3c05b 100644 --- a/pkg/scraper/timestamp_type.go +++ b/pkg/scraper/timestamp_type.go @@ -11,7 +11,7 @@ type Timestamp struct { } func (t Timestamp) Value() (driver.Value, error) { - return t.Unix(), nil + return t.UnixMilli(), nil } func (t *Timestamp) Scan(src interface{}) error { @@ -19,7 +19,7 @@ func (t *Timestamp) Scan(src interface{}) error { if !is_ok { return fmt.Errorf("Incompatible type for Timestamp: %#v", src) } - *t = Timestamp{time.Unix(val, 0)} + *t = Timestamp{time.UnixMilli(val)} return nil } @@ -38,3 +38,6 @@ func TimestampFromString(s string) (Timestamp, error) { func TimestampFromUnix(num int64) Timestamp { return Timestamp{time.Unix(num, 0)} } +func TimestampFromUnixMilli(num int64) Timestamp { + return Timestamp{time.UnixMilli(num)} +}