225 lines
6.3 KiB
Go
Raw Normal View History

2021-05-22 18:20:18 -04:00
package scraper
2021-06-13 14:34:20 -07:00
import (
"fmt"
2021-07-22 14:16:40 -07:00
"strings"
"regexp"
"path"
"offline_twitter/terminal_utils"
2021-06-13 14:34:20 -07:00
)
const DEFAULT_PROFILE_IMAGE_URL = "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
type UserID int64
2021-06-16 13:14:56 -07:00
type UserHandle string
2021-06-13 14:34:20 -07:00
2021-07-22 14:16:40 -07:00
func JoinArrayOfHandles(handles []UserHandle) string {
2021-06-16 19:31:27 -07:00
ret := []string{}
2021-07-22 14:16:40 -07:00
for _, h := range handles {
ret = append(ret, string(h))
2021-06-16 19:31:27 -07:00
}
2021-07-22 14:16:40 -07:00
return strings.Join(ret, ",")
2021-06-16 19:31:27 -07:00
}
2021-06-13 14:34:20 -07:00
type User struct {
ID UserID
DisplayName string
Handle UserHandle
Bio string
FollowingCount int
FollowersCount int
Location string
Website string
2022-03-06 18:09:43 -08:00
JoinDate Timestamp
IsPrivate bool
IsVerified bool
2022-01-06 14:39:31 -05:00
IsBanned bool
ProfileImageUrl string
ProfileImageLocalPath string
BannerImageUrl string
BannerImageLocalPath string
PinnedTweetID TweetID
PinnedTweet *Tweet
2021-08-07 16:51:38 -07:00
2022-02-26 15:58:30 -08:00
IsFollowed bool
2021-08-07 16:51:38 -07:00
IsContentDownloaded bool
IsNeedingFakeID bool
IsIdFake bool
2021-06-13 14:34:20 -07:00
}
func (u User) String() string {
var verified string
if u.IsVerified {
verified = "[\u2713]"
}
ret := fmt.Sprintf(
`%s%s
@%s
%s
Following: %d Followers: %d
Joined %s
%s
%s
`,
u.DisplayName,
verified,
u.Handle,
terminal_utils.WrapText(u.Bio, 60),
u.FollowingCount,
u.FollowersCount,
2022-03-06 18:09:43 -08:00
terminal_utils.FormatDate(u.JoinDate.Time),
u.Location,
u.Website,
)
if u.PinnedTweet != nil {
ret += "\n" + terminal_utils.WrapText(u.PinnedTweet.Text, 60)
} else {
println("Pinned tweet id:", u.PinnedTweetID)
}
return ret
2021-06-13 14:34:20 -07:00
}
2022-01-06 13:43:22 -05:00
2022-02-28 16:06:58 -08:00
/**
* Unknown Users with handles are only created by direct GetUser calls (either `twitter fetch_user`
* subcommand or as part of tombstone user fetching.)
*/
func GetUnknownUserWithHandle(handle UserHandle) User {
return User{
ID: UserID(0), // 2^62 + 1...
DisplayName: string(handle),
Handle: handle,
Bio: "<blank>",
FollowersCount: 0,
FollowingCount: 0,
Location: "<blank>",
Website:"<blank>",
2022-03-06 18:09:43 -08:00
JoinDate: TimestampFromUnix(0),
IsVerified: false,
2022-02-28 16:06:58 -08:00
IsPrivate: false,
IsNeedingFakeID: true,
IsIdFake: true,
}
}
2022-01-06 13:43:22 -05:00
2021-06-13 14:34:20 -07:00
// Turn an APIUser, as returned from the scraper, into a properly structured User object
func ParseSingleUser(apiUser APIUser) (ret User, err error) {
if apiUser.DoesntExist {
// User may have been deleted, or there was a typo. There's no data to parse
if apiUser.ScreenName == "" {
panic("ScreenName is empty!")
}
ret = GetUnknownUserWithHandle(UserHandle(apiUser.ScreenName))
return
}
ret.ID = UserID(apiUser.ID)
ret.Handle = UserHandle(apiUser.ScreenName)
2022-01-06 14:39:31 -05:00
if apiUser.IsBanned {
// Banned users won't have any further info, so just return here
ret.IsBanned = true
return
}
2021-06-13 14:34:20 -07:00
ret.DisplayName = apiUser.Name
ret.Bio = apiUser.Description
ret.FollowingCount = apiUser.FriendsCount
ret.FollowersCount = apiUser.FollowersCount
ret.Location = apiUser.Location
if len(apiUser.Entities.URL.Urls) > 0 {
ret.Website = apiUser.Entities.URL.Urls[0].ExpandedURL
}
2022-03-06 18:09:43 -08:00
ret.JoinDate, err = TimestampFromString(apiUser.CreatedAt)
2021-06-13 14:34:20 -07:00
if err != nil {
2022-03-06 19:27:30 -08:00
err = fmt.Errorf("Error parsing time on user ID %d: %w", ret.ID, err)
2021-06-13 14:34:20 -07:00
return
}
ret.IsPrivate = apiUser.Protected
ret.IsVerified = apiUser.Verified
ret.ProfileImageUrl = apiUser.ProfileImageURLHTTPS
2021-08-10 22:08:01 -07:00
if regexp.MustCompile(`_normal\.\w{2,4}`).MatchString(ret.ProfileImageUrl) {
ret.ProfileImageUrl = strings.ReplaceAll(ret.ProfileImageUrl, "_normal.", ".")
}
2021-06-13 14:34:20 -07:00
ret.BannerImageUrl = apiUser.ProfileBannerURL
ret.ProfileImageLocalPath = ret.compute_profile_image_local_path()
ret.BannerImageLocalPath = ret.compute_banner_image_local_path()
2021-06-13 14:34:20 -07:00
if len(apiUser.PinnedTweetIdsStr) > 0 {
ret.PinnedTweetID = TweetID(idstr_to_int(apiUser.PinnedTweetIdsStr[0]))
2021-06-13 14:34:20 -07:00
}
return
}
2021-06-16 19:31:27 -07:00
// Calls API#GetUser and returns the parsed result
func GetUser(handle UserHandle) (User, error) {
api := API{}
apiUser, err := api.GetUser(handle)
if apiUser.ScreenName == "" {
apiUser.ScreenName = string(handle)
}
2021-06-16 19:31:27 -07:00
if err != nil {
return User{}, err
}
return ParseSingleUser(apiUser)
}
/**
* Make a filename for the profile image, that hopefully won't clobber other ones
*/
func (u User) compute_profile_image_local_path() string {
return string(u.Handle) + "_profile_" + path.Base(u.ProfileImageUrl)
}
/**
* Make a filename for the banner image, that hopefully won't clobber other ones.
* Add a file extension if necessary (seems to be necessary).
* If there is no banner image, just return nothing.
*/
func (u User) compute_banner_image_local_path() string {
if u.BannerImageUrl == "" {
return ""
}
base_name := path.Base(u.BannerImageUrl)
// Check if it has an extension (e.g., ".png" or ".jpeg")
2021-08-10 22:08:01 -07:00
if !regexp.MustCompile(`\.\w{2,4}$`).MatchString(base_name) {
// If it doesn't have an extension, add one
base_name += ".jpg"
}
return string(u.Handle) + "_banner_" + base_name
}
/**
* Get the URL where we would expect to find a User's tiny profile image
*/
func (u User) GetTinyProfileImageUrl() string {
// If profile image is empty, then just use the default profile image
if u.ProfileImageUrl == "" {
return DEFAULT_PROFILE_IMAGE_URL
}
// Check that the format is as expected
r := regexp.MustCompile(`(\.\w{2,4})$`)
if !r.MatchString(u.ProfileImageUrl) {
2022-03-06 20:31:04 -08:00
panic(fmt.Errorf("Weird profile image url (here is the file extension?): %s", u.ProfileImageUrl))
}
return r.ReplaceAllString(u.ProfileImageUrl, "_normal$1")
}
/**
* If user has a profile image, return the local path for its tiny version.
* If user has a blank or default profile image, return a non-personalized default path.
*/
func (u User) GetTinyProfileImageLocalPath() string {
if u.ProfileImageUrl == "" {
return path.Base(u.GetTinyProfileImageUrl())
}
return string(u.Handle) + "_profile_" + path.Base(u.GetTinyProfileImageUrl())
}