diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index fe4c550..70d0477 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -51,6 +51,10 @@ func TestParseSingleTweet(t *testing.T) { if len(tweet.Mentions) != 1 || tweet.Mentions[0] != "michaelmalice" { t.Errorf("Expected %v, got %v", []string{"michaelmalice"}, tweet.Mentions) } + + if tweet.PostedAt.Unix() != 1621639105 { + t.Errorf("Expected %d, got %d", 1621639105, tweet.PostedAt.Unix()) + } } func TestParseSingleTweet2(t *testing.T) { diff --git a/scraper/user.go b/scraper/user.go index 428e60e..76abb4d 100644 --- a/scraper/user.go +++ b/scraper/user.go @@ -1,3 +1,55 @@ package scraper +import ( + "time" + "fmt" +) + type UserID string + +type User struct { + ID UserID + DisplayName string + Handle string + Bio string + FollowingCount int + FollowersCount int + Location string + Website string + JoinDate time.Time + IsPrivate bool + IsVerified bool + ProfileImageUrl string + BannerImageUrl string + PinnedTweet TweetID +} + +func (u User) String() string { + return fmt.Sprintf("%s (@%s)[%s]: %q", u.DisplayName, u.Handle, u.ID, u.Bio) +} + +// Turn an APIUser, as returned from the scraper, into a properly structured User object +func ParseSingleUser(apiUser APIUser) (ret User, err error) { + ret.ID = UserID(apiUser.IDStr) + ret.DisplayName = apiUser.Name + ret.Handle = apiUser.ScreenName + ret.Bio = apiUser.Description + ret.FollowingCount = apiUser.FriendsCount + ret.FollowersCount = apiUser.FollowersCount + ret.Location = apiUser.Location + if len(apiUser.Entities.URL.Urls) > 0 { + ret.Website = apiUser.Entities.URL.Urls[0].ExpandedURL + } + ret.JoinDate, err = time.Parse(time.RubyDate, apiUser.CreatedAt) + if err != nil { + return + } + ret.IsPrivate = apiUser.Protected + ret.IsVerified = apiUser.Verified + ret.ProfileImageUrl = apiUser.ProfileImageURLHTTPS + ret.BannerImageUrl = apiUser.ProfileBannerURL + if len(apiUser.PinnedTweetIdsStr) > 0 { + ret.PinnedTweet = TweetID(apiUser.PinnedTweetIdsStr[0]) + } + return +} diff --git a/scraper/user_test.go b/scraper/user_test.go new file mode 100644 index 0000000..372646d --- /dev/null +++ b/scraper/user_test.go @@ -0,0 +1,70 @@ +package scraper_test + +import ( + "encoding/json" + "io/ioutil" + "testing" + + "offline_twitter/scraper" +) + +func TestParseSingleUser(t *testing.T) { + data, err := ioutil.ReadFile("test_responses/michael_malice_user_profile.json") + if err != nil { + panic(err) + } + var user_resp scraper.UserResponse + err = json.Unmarshal(data, &user_resp) + if err != nil { + t.Errorf(err.Error()) + } + apiUser := user_resp.ConvertToAPIUser() + + user, err := scraper.ParseSingleUser(apiUser) + + if user.ID != "44067298" { + t.Errorf("Expected %q, got %q", "44067298", user.ID) + } + if user.DisplayName != "Michael Malice" { + t.Errorf("Expected %q, got %q", "Michael Malice", user.DisplayName) + } + if user.Handle != "michaelmalice" { + t.Errorf("Expected %q, got %q", "michaelmalice", user.Handle) + } + expectedBio := "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nUnderwear Model\nHe/Him ⚑" + if user.Bio != expectedBio { + t.Errorf("Expected %q, got %q", expectedBio, user.Bio) + } + if user.FollowingCount != 941 { + t.Errorf("Expected %d, got %d", 941, user.FollowingCount) + } + if user.FollowersCount != 208589 { + t.Errorf("Expected %d, got %d", 941, user.FollowersCount) + } + if user.Location != "Brooklyn" { + t.Errorf("Expected %q, got %q", "Brooklyn", user.Location) + } + if user.Website != "https://amzn.to/3oInafv" { + t.Errorf("Expected %q, got %q", "https://amzn.to/3oInafv", user.Website) + } + if user.JoinDate.Unix() != 1243920952 { + t.Errorf("Expected %d, got %d", 1243920952, user.JoinDate.Unix()) + } + if user.IsPrivate != false { + t.Errorf("Expected %v, got %v", false, user.IsPrivate) + } + if user.IsVerified != true { + t.Errorf("Expected %v, got %v", true, user.IsPrivate) + } + expectedProfileImage := "https://pbs.twimg.com/profile_images/1064051934812913664/Lbwdb_C9_normal.jpg" + if user.ProfileImageUrl != expectedProfileImage { + t.Errorf("Expected %q, got %q", expectedProfileImage, user.ProfileImageUrl) + } + expectedBannerImage := "https://pbs.twimg.com/profile_banners/44067298/1615134676" + if user.BannerImageUrl != expectedBannerImage { + t.Errorf("Expected %q, got %q", expectedBannerImage, user.BannerImageUrl) + } + if user.PinnedTweet != scraper.TweetID("1403835414373339136") { + t.Errorf("Expected %q, got %q", scraper.TweetID("1403835414373339136"), user.PinnedTweet) + } +}